node { name: "global_step/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@global_step" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT64 } } attr { key: "value" value { tensor { dtype: DT_INT64 tensor_shape { } int64_val: 0 } } } } node { name: "global_step" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@global_step" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_INT64 } } attr { key: "shape" value { shape { } } } attr { key: "shared_name" value { s: "global_step" } } } node { name: "global_step/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "global_step" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_step/Assign" op: "AssignVariableOp" input: "global_step" input: "global_step/Initializer/zeros" attr { key: "_class" value { list { s: "loc:@global_step" } } } attr { key: "dtype" value { type: DT_INT64 } } } node { name: "global_step/Read/ReadVariableOp" op: "ReadVariableOp" input: "global_step" attr { key: "_class" value { list { s: "loc:@global_step" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT64 } } } node { name: "global_step/VarIsInitializedOp" op: "VarIsInitializedOp" input: "global_step" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_step/cond/Switch" op: "Switch" input: "global_step/VarIsInitializedOp" input: "global_step/VarIsInitializedOp" attr { key: "T" value { type: DT_BOOL } } attr { key: "_output_shapes" value { list { shape { } shape { } } } } } node { name: "global_step/cond/switch_t" op: "Identity" input: "global_step/cond/Switch:1" attr { key: "T" value { type: DT_BOOL } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_step/cond/switch_f" op: "Identity" input: "global_step/cond/Switch" attr { key: "T" value { type: DT_BOOL } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_step/cond/pred_id" op: "Identity" input: "global_step/VarIsInitializedOp" attr { key: "T" value { type: DT_BOOL } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_step/cond/Read/ReadVariableOp" op: "ReadVariableOp" input: "global_step/cond/Read/ReadVariableOp/Switch:1" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT64 } } } node { name: "global_step/cond/Read/ReadVariableOp/Switch" op: "Switch" input: "global_step" input: "global_step/cond/pred_id" attr { key: "T" value { type: DT_RESOURCE } } attr { key: "_class" value { list { s: "loc:@global_step" } } } attr { key: "_output_shapes" value { list { shape { } shape { } } } } } node { name: "global_step/cond/Identity" op: "Identity" input: "global_step/cond/Read/ReadVariableOp" attr { key: "T" value { type: DT_INT64 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_step/cond/Switch_1" op: "Switch" input: "global_step/Initializer/zeros" input: "global_step/cond/pred_id" attr { key: "T" value { type: DT_INT64 } } attr { key: "_class" value { list { s: "loc:@global_step" } } } attr { key: "_output_shapes" value { list { shape { } shape { } } } } } node { name: "global_step/cond/Merge" op: "Merge" input: "global_step/cond/Switch_1" input: "global_step/cond/Identity" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT64 } } attr { key: "_output_shapes" value { list { shape { } shape { } } } } } node { name: "global_step/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT64 } } attr { key: "value" value { tensor { dtype: DT_INT64 tensor_shape { } int64_val: 0 } } } } node { name: "global_step/add" op: "Add" input: "global_step/cond/Merge" input: "global_step/add/y" attr { key: "T" value { type: DT_INT64 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "Const" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "../data/tf_records/discharge_summary.tfrecord" } } } } node { name: "count" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT64 } } attr { key: "value" value { tensor { dtype: DT_INT64 tensor_shape { } int64_val: -1 } } } } node { name: "buffer_size" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT64 } } attr { key: "value" value { tensor { dtype: DT_INT64 tensor_shape { } int64_val: 1 } } } } node { name: "seed" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT64 } } attr { key: "value" value { tensor { dtype: DT_INT64 tensor_shape { } int64_val: 0 } } } } node { name: "seed2" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT64 } } attr { key: "value" value { tensor { dtype: DT_INT64 tensor_shape { } int64_val: 0 } } } } node { name: "cycle_length" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT64 } } attr { key: "value" value { tensor { dtype: DT_INT64 tensor_shape { } int64_val: 1 } } } } node { name: "block_length" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT64 } } attr { key: "value" value { tensor { dtype: DT_INT64 tensor_shape { } int64_val: 1 } } } } node { name: "sloppy" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_BOOL } } attr { key: "value" value { tensor { dtype: DT_BOOL tensor_shape { } bool_val: true } } } } node { name: "buffer_output_elements" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT64 } } attr { key: "value" value { tensor { dtype: DT_INT64 tensor_shape { } int64_val: 2 } } } } node { name: "prefetch_input_elements" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT64 } } attr { key: "value" value { tensor { dtype: DT_INT64 tensor_shape { } int64_val: 2 } } } } node { name: "buffer_size_1" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT64 } } attr { key: "value" value { tensor { dtype: DT_INT64 tensor_shape { } int64_val: 100 } } } } node { name: "seed_1" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT64 } } attr { key: "value" value { tensor { dtype: DT_INT64 tensor_shape { } int64_val: 0 } } } } node { name: "seed2_1" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT64 } } attr { key: "value" value { tensor { dtype: DT_INT64 tensor_shape { } int64_val: 0 } } } } node { name: "batch_size" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT64 } } attr { key: "value" value { tensor { dtype: DT_INT64 tensor_shape { } int64_val: 32 } } } } node { name: "num_parallel_calls" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT64 } } attr { key: "value" value { tensor { dtype: DT_INT64 tensor_shape { } int64_val: 128 } } } } node { name: "drop_remainder" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_BOOL } } attr { key: "value" value { tensor { dtype: DT_BOOL tensor_shape { } bool_val: true } } } } node { name: "IteratorV2" op: "IteratorV2" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "container" value { s: "" } } attr { key: "output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } } shape { dim { size: 32 } dim { size: 128 } } shape { dim { size: 32 } dim { size: 20 } } shape { dim { size: 32 } dim { size: 20 } } shape { dim { size: 32 } dim { size: 20 } } shape { dim { size: 32 } dim { size: 1 } } shape { dim { size: 32 } dim { size: 128 } } } } } attr { key: "output_types" value { list { type: DT_INT32 type: DT_INT32 type: DT_INT32 type: DT_INT32 type: DT_FLOAT type: DT_INT32 type: DT_INT32 } } } attr { key: "shared_name" value { s: "" } } } node { name: "TensorSliceDataset" op: "TensorSliceDataset" input: "Const" attr { key: "Toutput_types" value { list { type: DT_STRING } } } attr { key: "_class" value { list { s: "loc:@IteratorV2" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "output_shapes" value { list { shape { } } } } } node { name: "RepeatDataset" op: "RepeatDataset" input: "TensorSliceDataset" input: "count" attr { key: "_class" value { list { s: "loc:@IteratorV2" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "output_shapes" value { list { shape { } } } } attr { key: "output_types" value { list { type: DT_STRING } } } } node { name: "ShuffleDataset" op: "ShuffleDataset" input: "RepeatDataset" input: "buffer_size" input: "seed" input: "seed2" attr { key: "_class" value { list { s: "loc:@IteratorV2" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "output_shapes" value { list { shape { } } } } attr { key: "output_types" value { list { type: DT_STRING } } } attr { key: "reshuffle_each_iteration" value { b: true } } } node { name: "ParallelInterleaveDataset" op: "ParallelInterleaveDataset" input: "ShuffleDataset" input: "cycle_length" input: "block_length" input: "sloppy" input: "buffer_output_elements" input: "prefetch_input_elements" attr { key: "Targuments" value { list { } } } attr { key: "_class" value { list { s: "loc:@IteratorV2" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "f" value { func { name: "tf_data_structured_function_wrapper_rn3e6kArW78" } } } attr { key: "output_shapes" value { list { shape { } } } } attr { key: "output_types" value { list { type: DT_STRING } } } } node { name: "ShuffleDataset_1" op: "ShuffleDataset" input: "ParallelInterleaveDataset" input: "buffer_size_1" input: "seed_1" input: "seed2_1" attr { key: "_class" value { list { s: "loc:@IteratorV2" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "output_shapes" value { list { shape { } } } } attr { key: "output_types" value { list { type: DT_STRING } } } attr { key: "reshuffle_each_iteration" value { b: true } } } node { name: "MapAndBatchDatasetV2" op: "MapAndBatchDatasetV2" input: "ShuffleDataset_1" input: "batch_size" input: "num_parallel_calls" input: "drop_remainder" attr { key: "Targuments" value { list { } } } attr { key: "_class" value { list { s: "loc:@IteratorV2" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "f" value { func { name: "tf_data_structured_function_wrapper_9z4XDFOWUdQ" } } } attr { key: "output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } } shape { dim { size: 32 } dim { size: 128 } } shape { dim { size: 32 } dim { size: 20 } } shape { dim { size: 32 } dim { size: 20 } } shape { dim { size: 32 } dim { size: 20 } } shape { dim { size: 32 } dim { size: 1 } } shape { dim { size: 32 } dim { size: 128 } } } } } attr { key: "output_types" value { list { type: DT_INT32 type: DT_INT32 type: DT_INT32 type: DT_INT32 type: DT_FLOAT type: DT_INT32 type: DT_INT32 } } } } node { name: "MakeIterator" op: "MakeIterator" input: "MapAndBatchDatasetV2" input: "IteratorV2" attr { key: "_class" value { list { s: "loc:@IteratorV2" } } } } node { name: "IteratorToStringHandle" op: "IteratorToStringHandle" input: "IteratorV2" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "IteratorGetNext" op: "IteratorGetNext" input: "IteratorV2" attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } } shape { dim { size: 32 } dim { size: 128 } } shape { dim { size: 32 } dim { size: 20 } } shape { dim { size: 32 } dim { size: 20 } } shape { dim { size: 32 } dim { size: 20 } } shape { dim { size: 32 } dim { size: 1 } } shape { dim { size: 32 } dim { size: 128 } } } } } attr { key: "output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } } shape { dim { size: 32 } dim { size: 128 } } shape { dim { size: 32 } dim { size: 20 } } shape { dim { size: 32 } dim { size: 20 } } shape { dim { size: 32 } dim { size: 20 } } shape { dim { size: 32 } dim { size: 1 } } shape { dim { size: 32 } dim { size: 128 } } } } } attr { key: "output_types" value { list { type: DT_INT32 type: DT_INT32 type: DT_INT32 type: DT_INT32 type: DT_FLOAT type: DT_INT32 type: DT_INT32 } } } } node { name: "bert/embeddings/ExpandDims/dim" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: -1 } } } } node { name: "bert/embeddings/ExpandDims" op: "ExpandDims" input: "IteratorGetNext" input: "bert/embeddings/ExpandDims/dim" attr { key: "T" value { type: DT_INT32 } } attr { key: "Tdim" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 1 } } } } } } node { name: "bert/embeddings/word_embeddings/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "Dq\000\000\000\003\000\000" } } } } node { name: "bert/embeddings/word_embeddings/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/embeddings/word_embeddings/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/embeddings/word_embeddings/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/embeddings/word_embeddings/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/embeddings/word_embeddings/Initializer/truncated_normal/mul" op: "Mul" input: "bert/embeddings/word_embeddings/Initializer/truncated_normal/TruncatedNormal" input: "bert/embeddings/word_embeddings/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } dim { size: 768 } } } } } } node { name: "bert/embeddings/word_embeddings/Initializer/truncated_normal" op: "Add" input: "bert/embeddings/word_embeddings/Initializer/truncated_normal/mul" input: "bert/embeddings/word_embeddings/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } dim { size: 768 } } } } } } node { name: "bert/embeddings/word_embeddings" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 28996 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/embeddings/word_embeddings/Assign" op: "Assign" input: "bert/embeddings/word_embeddings" input: "bert/embeddings/word_embeddings/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/embeddings/word_embeddings/read" op: "Identity" input: "bert/embeddings/word_embeddings" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } dim { size: 768 } } } } } } node { name: "bert/embeddings/embedding_lookup/axis" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "bert/embeddings/embedding_lookup" op: "GatherV2" input: "bert/embeddings/word_embeddings/read" input: "bert/embeddings/ExpandDims" input: "bert/embeddings/embedding_lookup/axis" attr { key: "Taxis" value { type: DT_INT32 } } attr { key: "Tindices" value { type: DT_INT32 } } attr { key: "Tparams" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 1 } dim { size: 768 } } } } } } node { name: "bert/embeddings/embedding_lookup/Identity" op: "Identity" input: "bert/embeddings/embedding_lookup" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 1 } dim { size: 768 } } } } } } node { name: "bert/embeddings/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: " \000\000\000\200\000\000\000\000\003\000\000" } } } } node { name: "bert/embeddings/Reshape" op: "Reshape" input: "bert/embeddings/embedding_lookup/Identity" input: "bert/embeddings/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "bert/embeddings/token_type_embeddings/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\002\000\000\000\000\003\000\000" } } } } node { name: "bert/embeddings/token_type_embeddings/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/embeddings/token_type_embeddings/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/embeddings/token_type_embeddings/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/embeddings/token_type_embeddings/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/embeddings/token_type_embeddings/Initializer/truncated_normal/mul" op: "Mul" input: "bert/embeddings/token_type_embeddings/Initializer/truncated_normal/TruncatedNormal" input: "bert/embeddings/token_type_embeddings/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "bert/embeddings/token_type_embeddings/Initializer/truncated_normal" op: "Add" input: "bert/embeddings/token_type_embeddings/Initializer/truncated_normal/mul" input: "bert/embeddings/token_type_embeddings/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "bert/embeddings/token_type_embeddings" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 2 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/embeddings/token_type_embeddings/Assign" op: "Assign" input: "bert/embeddings/token_type_embeddings" input: "bert/embeddings/token_type_embeddings/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/embeddings/token_type_embeddings/read" op: "Identity" input: "bert/embeddings/token_type_embeddings" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "bert/embeddings/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: -1 } } } } node { name: "bert/embeddings/Reshape_1" op: "Reshape" input: "IteratorGetNext:6" input: "bert/embeddings/Reshape_1/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } } node { name: "bert/embeddings/one_hot/on_value" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/embeddings/one_hot/off_value" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/embeddings/one_hot/depth" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "bert/embeddings/one_hot" op: "OneHot" input: "bert/embeddings/Reshape_1" input: "bert/embeddings/one_hot/depth" input: "bert/embeddings/one_hot/on_value" input: "bert/embeddings/one_hot/off_value" attr { key: "T" value { type: DT_FLOAT } } attr { key: "TI" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 2 } } } } } attr { key: "axis" value { i: -1 } } } node { name: "bert/embeddings/MatMul" op: "MatMul" input: "bert/embeddings/one_hot" input: "bert/embeddings/token_type_embeddings/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/embeddings/Reshape_2/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: " \000\000\000\200\000\000\000\000\003\000\000" } } } } node { name: "bert/embeddings/Reshape_2" op: "Reshape" input: "bert/embeddings/MatMul" input: "bert/embeddings/Reshape_2/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "bert/embeddings/add" op: "Add" input: "bert/embeddings/Reshape" input: "bert/embeddings/Reshape_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "bert/embeddings/assert_less_equal/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 128 } } } } node { name: "bert/embeddings/assert_less_equal/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 512 } } } } node { name: "bert/embeddings/assert_less_equal/LessEqual" op: "LessEqual" input: "bert/embeddings/assert_less_equal/x" input: "bert/embeddings/assert_less_equal/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/embeddings/assert_less_equal/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "bert/embeddings/assert_less_equal/All" op: "All" input: "bert/embeddings/assert_less_equal/LessEqual" input: "bert/embeddings/assert_less_equal/Const" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "bert/embeddings/assert_less_equal/Assert/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "" } } } } node { name: "bert/embeddings/assert_less_equal/Assert/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "Condition x <= y did not hold element-wise:x (bert/embeddings/assert_less_equal/x:0) = " } } } } node { name: "bert/embeddings/assert_less_equal/Assert/Const_2" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "y (bert/embeddings/assert_less_equal/y:0) = " } } } } node { name: "bert/embeddings/assert_less_equal/Assert/Assert/data_0" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "" } } } } node { name: "bert/embeddings/assert_less_equal/Assert/Assert/data_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "Condition x <= y did not hold element-wise:x (bert/embeddings/assert_less_equal/x:0) = " } } } } node { name: "bert/embeddings/assert_less_equal/Assert/Assert/data_3" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "y (bert/embeddings/assert_less_equal/y:0) = " } } } } node { name: "bert/embeddings/assert_less_equal/Assert/Assert" op: "Assert" input: "bert/embeddings/assert_less_equal/All" input: "bert/embeddings/assert_less_equal/Assert/Assert/data_0" input: "bert/embeddings/assert_less_equal/Assert/Assert/data_1" input: "bert/embeddings/assert_less_equal/x" input: "bert/embeddings/assert_less_equal/Assert/Assert/data_3" input: "bert/embeddings/assert_less_equal/y" attr { key: "T" value { list { type: DT_STRING type: DT_STRING type: DT_INT32 type: DT_STRING type: DT_INT32 } } } attr { key: "summarize" value { i: 3 } } } node { name: "bert/embeddings/position_embeddings/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\002\000\000\000\003\000\000" } } } } node { name: "bert/embeddings/position_embeddings/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/embeddings/position_embeddings/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/embeddings/position_embeddings/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/embeddings/position_embeddings/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/embeddings/position_embeddings/Initializer/truncated_normal/mul" op: "Mul" input: "bert/embeddings/position_embeddings/Initializer/truncated_normal/TruncatedNormal" input: "bert/embeddings/position_embeddings/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } } node { name: "bert/embeddings/position_embeddings/Initializer/truncated_normal" op: "Add" input: "bert/embeddings/position_embeddings/Initializer/truncated_normal/mul" input: "bert/embeddings/position_embeddings/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } } node { name: "bert/embeddings/position_embeddings" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 512 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/embeddings/position_embeddings/Assign" op: "Assign" input: "bert/embeddings/position_embeddings" input: "bert/embeddings/position_embeddings/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/embeddings/position_embeddings/read" op: "Identity" input: "bert/embeddings/position_embeddings" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } } node { name: "bert/embeddings/Slice/begin" op: "Const" input: "^bert/embeddings/assert_less_equal/Assert/Assert" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\000\000\000\000" } } } } node { name: "bert/embeddings/Slice/size" op: "Const" input: "^bert/embeddings/assert_less_equal/Assert/Assert" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\200\000\000\000\377\377\377\377" } } } } node { name: "bert/embeddings/Slice" op: "Slice" input: "bert/embeddings/position_embeddings/read" input: "bert/embeddings/Slice/begin" input: "bert/embeddings/Slice/size" attr { key: "Index" value { type: DT_INT32 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 128 } dim { size: 768 } } } } } } node { name: "bert/embeddings/Reshape_3/shape" op: "Const" input: "^bert/embeddings/assert_less_equal/Assert/Assert" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\001\000\000\000\200\000\000\000\000\003\000\000" } } } } node { name: "bert/embeddings/Reshape_3" op: "Reshape" input: "bert/embeddings/Slice" input: "bert/embeddings/Reshape_3/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "bert/embeddings/add_1" op: "Add" input: "bert/embeddings/add" input: "bert/embeddings/Reshape_3" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "bert/embeddings/LayerNorm/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/embeddings/LayerNorm/beta" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/embeddings/LayerNorm/beta/Assign" op: "Assign" input: "bert/embeddings/LayerNorm/beta" input: "bert/embeddings/LayerNorm/beta/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/embeddings/LayerNorm/beta/read" op: "Identity" input: "bert/embeddings/LayerNorm/beta" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/embeddings/LayerNorm/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/embeddings/LayerNorm/gamma" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/embeddings/LayerNorm/gamma/Assign" op: "Assign" input: "bert/embeddings/LayerNorm/gamma" input: "bert/embeddings/LayerNorm/gamma/Initializer/ones" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/embeddings/LayerNorm/gamma/read" op: "Identity" input: "bert/embeddings/LayerNorm/gamma" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/embeddings/LayerNorm/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 2 } } } } node { name: "bert/embeddings/LayerNorm/moments/mean" op: "Mean" input: "bert/embeddings/add_1" input: "bert/embeddings/LayerNorm/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/embeddings/LayerNorm/moments/StopGradient" op: "StopGradient" input: "bert/embeddings/LayerNorm/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 1 } } } } } } node { name: "bert/embeddings/LayerNorm/moments/SquaredDifference" op: "SquaredDifference" input: "bert/embeddings/add_1" input: "bert/embeddings/LayerNorm/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "bert/embeddings/LayerNorm/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 2 } } } } node { name: "bert/embeddings/LayerNorm/moments/variance" op: "Mean" input: "bert/embeddings/LayerNorm/moments/SquaredDifference" input: "bert/embeddings/LayerNorm/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/embeddings/LayerNorm/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/embeddings/LayerNorm/batchnorm/add" op: "Add" input: "bert/embeddings/LayerNorm/moments/variance" input: "bert/embeddings/LayerNorm/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 1 } } } } } } node { name: "bert/embeddings/LayerNorm/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/embeddings/LayerNorm/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 1 } } } } } } node { name: "bert/embeddings/LayerNorm/batchnorm/mul" op: "Mul" input: "bert/embeddings/LayerNorm/batchnorm/Rsqrt" input: "bert/embeddings/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "bert/embeddings/LayerNorm/batchnorm/mul_1" op: "Mul" input: "bert/embeddings/add_1" input: "bert/embeddings/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "bert/embeddings/LayerNorm/batchnorm/mul_2" op: "Mul" input: "bert/embeddings/LayerNorm/moments/mean" input: "bert/embeddings/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "bert/embeddings/LayerNorm/batchnorm/sub" op: "Sub" input: "bert/embeddings/LayerNorm/beta/read" input: "bert/embeddings/LayerNorm/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "bert/embeddings/LayerNorm/batchnorm/add_1" op: "Add" input: "bert/embeddings/LayerNorm/batchnorm/mul_1" input: "bert/embeddings/LayerNorm/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "bert/embeddings/dropout/keep_prob" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "bert/embeddings/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: " \000\000\000\200\000\000\000\000\003\000\000" } } } } node { name: "bert/embeddings/dropout/random_uniform/min" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/embeddings/dropout/random_uniform/max" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/embeddings/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/embeddings/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/embeddings/dropout/random_uniform/sub" op: "Sub" input: "bert/embeddings/dropout/random_uniform/max" input: "bert/embeddings/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/embeddings/dropout/random_uniform/mul" op: "Mul" input: "bert/embeddings/dropout/random_uniform/RandomUniform" input: "bert/embeddings/dropout/random_uniform/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "bert/embeddings/dropout/random_uniform" op: "Add" input: "bert/embeddings/dropout/random_uniform/mul" input: "bert/embeddings/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "bert/embeddings/dropout/add" op: "Add" input: "bert/embeddings/dropout/keep_prob" input: "bert/embeddings/dropout/random_uniform" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "bert/embeddings/dropout/Floor" op: "Floor" input: "bert/embeddings/dropout/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "bert/embeddings/dropout/div" op: "RealDiv" input: "bert/embeddings/LayerNorm/batchnorm/add_1" input: "bert/embeddings/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "bert/embeddings/dropout/mul" op: "Mul" input: "bert/embeddings/dropout/div" input: "bert/embeddings/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "bert/encoder/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: " \000\000\000\001\000\000\000\200\000\000\000" } } } } node { name: "bert/encoder/Reshape" op: "Reshape" input: "IteratorGetNext:1" input: "bert/encoder/Reshape/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } } } } } } node { name: "bert/encoder/Cast" op: "Cast" input: "bert/encoder/Reshape" attr { key: "DstT" value { type: DT_FLOAT } } attr { key: "SrcT" value { type: DT_INT32 } } attr { key: "Truncate" value { b: false } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } } } } } } node { name: "bert/encoder/ones/shape_as_tensor" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: " \000\000\000\200\000\000\000\001\000\000\000" } } } } node { name: "bert/encoder/ones/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/ones" op: "Fill" input: "bert/encoder/ones/shape_as_tensor" input: "bert/encoder/ones/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/mul" op: "Mul" input: "bert/encoder/ones" input: "bert/encoder/Cast" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\377\377\377\377\000\003\000\000" } } } } node { name: "bert/encoder/Reshape_1" op: "Reshape" input: "bert/embeddings/dropout/mul" input: "bert/encoder/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/Assign" op: "Assign" input: "bert/encoder/layer_0/attention/self/query/kernel" input: "bert/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/read" op: "Identity" input: "bert/encoder/layer_0/attention/self/query/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/self/query/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/self/query/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/attention/self/query/bias/Assign" op: "Assign" input: "bert/encoder/layer_0/attention/self/query/bias" input: "bert/encoder/layer_0/attention/self/query/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/attention/self/query/bias/read" op: "Identity" input: "bert/encoder/layer_0/attention/self/query/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/self/query/MatMul" op: "MatMul" input: "bert/encoder/Reshape_1" input: "bert/encoder/layer_0/attention/self/query/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_0/attention/self/query/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_0/attention/self/query/MatMul" input: "bert/encoder/layer_0/attention/self/query/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/Assign" op: "Assign" input: "bert/encoder/layer_0/attention/self/key/kernel" input: "bert/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/read" op: "Identity" input: "bert/encoder/layer_0/attention/self/key/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/self/key/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/self/key/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/attention/self/key/bias/Assign" op: "Assign" input: "bert/encoder/layer_0/attention/self/key/bias" input: "bert/encoder/layer_0/attention/self/key/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/attention/self/key/bias/read" op: "Identity" input: "bert/encoder/layer_0/attention/self/key/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/self/key/MatMul" op: "MatMul" input: "bert/encoder/Reshape_1" input: "bert/encoder/layer_0/attention/self/key/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_0/attention/self/key/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_0/attention/self/key/MatMul" input: "bert/encoder/layer_0/attention/self/key/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/Assign" op: "Assign" input: "bert/encoder/layer_0/attention/self/value/kernel" input: "bert/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/read" op: "Identity" input: "bert/encoder/layer_0/attention/self/value/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/self/value/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/self/value/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/attention/self/value/bias/Assign" op: "Assign" input: "bert/encoder/layer_0/attention/self/value/bias" input: "bert/encoder/layer_0/attention/self/value/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/attention/self/value/bias/read" op: "Identity" input: "bert/encoder/layer_0/attention/self/value/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/self/value/MatMul" op: "MatMul" input: "bert/encoder/Reshape_1" input: "bert/encoder/layer_0/attention/self/value/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_0/attention/self/value/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_0/attention/self/value/MatMul" input: "bert/encoder/layer_0/attention/self/value/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_0/attention/self/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_0/attention/self/Reshape" op: "Reshape" input: "bert/encoder/layer_0/attention/self/query/BiasAdd" input: "bert/encoder/layer_0/attention/self/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_0/attention/self/transpose/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_0/attention/self/transpose" op: "Transpose" input: "bert/encoder/layer_0/attention/self/Reshape" input: "bert/encoder/layer_0/attention/self/transpose/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_0/attention/self/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_0/attention/self/Reshape_1" op: "Reshape" input: "bert/encoder/layer_0/attention/self/key/BiasAdd" input: "bert/encoder/layer_0/attention/self/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_0/attention/self/transpose_1/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_0/attention/self/transpose_1" op: "Transpose" input: "bert/encoder/layer_0/attention/self/Reshape_1" input: "bert/encoder/layer_0/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_0/attention/self/MatMul" op: "BatchMatMul" input: "bert/encoder/layer_0/attention/self/transpose" input: "bert/encoder/layer_0/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "bert/encoder/layer_0/attention/self/Mul/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.125 } } } } node { name: "bert/encoder/layer_0/attention/self/Mul" op: "Mul" input: "bert/encoder/layer_0/attention/self/MatMul" input: "bert/encoder/layer_0/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_0/attention/self/ExpandDims/dim" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_0/attention/self/ExpandDims" op: "ExpandDims" input: "bert/encoder/mul" input: "bert/encoder/layer_0/attention/self/ExpandDims/dim" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tdim" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_0/attention/self/sub/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_0/attention/self/sub" op: "Sub" input: "bert/encoder/layer_0/attention/self/sub/x" input: "bert/encoder/layer_0/attention/self/ExpandDims" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_0/attention/self/mul_1/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: -10000.0 } } } } node { name: "bert/encoder/layer_0/attention/self/mul_1" op: "Mul" input: "bert/encoder/layer_0/attention/self/sub" input: "bert/encoder/layer_0/attention/self/mul_1/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_0/attention/self/add" op: "Add" input: "bert/encoder/layer_0/attention/self/Mul" input: "bert/encoder/layer_0/attention/self/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_0/attention/self/Softmax" op: "Softmax" input: "bert/encoder/layer_0/attention/self/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_0/attention/self/dropout/keep_prob" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "bert/encoder/layer_0/attention/self/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "bert/encoder/layer_0/attention/self/dropout/random_uniform/min" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/self/dropout/random_uniform/max" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_0/attention/self/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_0/attention/self/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_0/attention/self/dropout/random_uniform/sub" op: "Sub" input: "bert/encoder/layer_0/attention/self/dropout/random_uniform/max" input: "bert/encoder/layer_0/attention/self/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/attention/self/dropout/random_uniform/mul" op: "Mul" input: "bert/encoder/layer_0/attention/self/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_0/attention/self/dropout/random_uniform/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_0/attention/self/dropout/random_uniform" op: "Add" input: "bert/encoder/layer_0/attention/self/dropout/random_uniform/mul" input: "bert/encoder/layer_0/attention/self/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_0/attention/self/dropout/add" op: "Add" input: "bert/encoder/layer_0/attention/self/dropout/keep_prob" input: "bert/encoder/layer_0/attention/self/dropout/random_uniform" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_0/attention/self/dropout/Floor" op: "Floor" input: "bert/encoder/layer_0/attention/self/dropout/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_0/attention/self/dropout/div" op: "RealDiv" input: "bert/encoder/layer_0/attention/self/Softmax" input: "bert/encoder/layer_0/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_0/attention/self/dropout/mul" op: "Mul" input: "bert/encoder/layer_0/attention/self/dropout/div" input: "bert/encoder/layer_0/attention/self/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_0/attention/self/Reshape_2/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_0/attention/self/Reshape_2" op: "Reshape" input: "bert/encoder/layer_0/attention/self/value/BiasAdd" input: "bert/encoder/layer_0/attention/self/Reshape_2/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_0/attention/self/transpose_2/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_0/attention/self/transpose_2" op: "Transpose" input: "bert/encoder/layer_0/attention/self/Reshape_2" input: "bert/encoder/layer_0/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_0/attention/self/MatMul_1" op: "BatchMatMul" input: "bert/encoder/layer_0/attention/self/dropout/mul" input: "bert/encoder/layer_0/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "bert/encoder/layer_0/attention/self/transpose_3/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_0/attention/self/transpose_3" op: "Transpose" input: "bert/encoder/layer_0/attention/self/MatMul_1" input: "bert/encoder/layer_0/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_0/attention/self/Reshape_3/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_0/attention/self/Reshape_3" op: "Reshape" input: "bert/encoder/layer_0/attention/self/transpose_3" input: "bert/encoder/layer_0/attention/self/Reshape_3/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/Assign" op: "Assign" input: "bert/encoder/layer_0/attention/output/dense/kernel" input: "bert/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/read" op: "Identity" input: "bert/encoder/layer_0/attention/output/dense/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/output/dense/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/attention/output/dense/bias/Assign" op: "Assign" input: "bert/encoder/layer_0/attention/output/dense/bias" input: "bert/encoder/layer_0/attention/output/dense/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/attention/output/dense/bias/read" op: "Identity" input: "bert/encoder/layer_0/attention/output/dense/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_0/attention/self/Reshape_3" input: "bert/encoder/layer_0/attention/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_0/attention/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_0/attention/output/dense/MatMul" input: "bert/encoder/layer_0/attention/output/dense/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_0/attention/output/dropout/keep_prob" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "bert/encoder/layer_0/attention/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_0/attention/output/dropout/random_uniform/min" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/output/dropout/random_uniform/max" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_0/attention/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_0/attention/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_0/attention/output/dropout/random_uniform/sub" op: "Sub" input: "bert/encoder/layer_0/attention/output/dropout/random_uniform/max" input: "bert/encoder/layer_0/attention/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/attention/output/dropout/random_uniform/mul" op: "Mul" input: "bert/encoder/layer_0/attention/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_0/attention/output/dropout/random_uniform/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/output/dropout/random_uniform" op: "Add" input: "bert/encoder/layer_0/attention/output/dropout/random_uniform/mul" input: "bert/encoder/layer_0/attention/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/output/dropout/add" op: "Add" input: "bert/encoder/layer_0/attention/output/dropout/keep_prob" input: "bert/encoder/layer_0/attention/output/dropout/random_uniform" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/output/dropout/Floor" op: "Floor" input: "bert/encoder/layer_0/attention/output/dropout/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/output/dropout/div" op: "RealDiv" input: "bert/encoder/layer_0/attention/output/dense/BiasAdd" input: "bert/encoder/layer_0/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/output/dropout/mul" op: "Mul" input: "bert/encoder/layer_0/attention/output/dropout/div" input: "bert/encoder/layer_0/attention/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/output/add" op: "Add" input: "bert/encoder/layer_0/attention/output/dropout/mul" input: "bert/encoder/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/output/LayerNorm/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/output/LayerNorm/beta" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/attention/output/LayerNorm/beta/Assign" op: "Assign" input: "bert/encoder/layer_0/attention/output/LayerNorm/beta" input: "bert/encoder/layer_0/attention/output/LayerNorm/beta/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/attention/output/LayerNorm/beta/read" op: "Identity" input: "bert/encoder/layer_0/attention/output/LayerNorm/beta" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_0/attention/output/LayerNorm/gamma" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/Assign" op: "Assign" input: "bert/encoder/layer_0/attention/output/LayerNorm/gamma" input: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/Initializer/ones" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/read" op: "Identity" input: "bert/encoder/layer_0/attention/output/LayerNorm/gamma" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/output/LayerNorm/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_0/attention/output/LayerNorm/moments/mean" op: "Mean" input: "bert/encoder/layer_0/attention/output/add" input: "bert/encoder/layer_0/attention/output/LayerNorm/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_0/attention/output/LayerNorm/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_0/attention/output/LayerNorm/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_0/attention/output/add" input: "bert/encoder/layer_0/attention/output/LayerNorm/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/output/LayerNorm/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_0/attention/output/LayerNorm/moments/variance" op: "Mean" input: "bert/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference" input: "bert/encoder/layer_0/attention/output/LayerNorm/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/add" op: "Add" input: "bert/encoder/layer_0/attention/output/LayerNorm/moments/variance" input: "bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/Rsqrt" input: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_0/attention/output/add" input: "bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_0/attention/output/LayerNorm/moments/mean" input: "bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_0/attention/output/LayerNorm/beta/read" input: "bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_1" op: "Add" input: "bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_1" input: "bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/Assign" op: "Assign" input: "bert/encoder/layer_0/intermediate/dense/kernel" input: "bert/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/read" op: "Identity" input: "bert/encoder/layer_0/intermediate/dense/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_0/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_0/intermediate/dense/bias/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/intermediate/dense/bias/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_0/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_0/intermediate/dense/bias/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_0/intermediate/dense/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/intermediate/dense/bias/Assign" op: "Assign" input: "bert/encoder/layer_0/intermediate/dense/bias" input: "bert/encoder/layer_0/intermediate/dense/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/intermediate/dense/bias/read" op: "Identity" input: "bert/encoder/layer_0/intermediate/dense/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_0/intermediate/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_0/intermediate/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_0/intermediate/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_0/intermediate/dense/MatMul" input: "bert/encoder/layer_0/intermediate/dense/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_0/intermediate/dense/Sqrt/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "bert/encoder/layer_0/intermediate/dense/Sqrt" op: "Sqrt" input: "bert/encoder/layer_0/intermediate/dense/Sqrt/x" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/intermediate/dense/truediv" op: "RealDiv" input: "bert/encoder/layer_0/intermediate/dense/BiasAdd" input: "bert/encoder/layer_0/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_0/intermediate/dense/Erf" op: "Erf" input: "bert/encoder/layer_0/intermediate/dense/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_0/intermediate/dense/add/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_0/intermediate/dense/add" op: "Add" input: "bert/encoder/layer_0/intermediate/dense/add/x" input: "bert/encoder/layer_0/intermediate/dense/Erf" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_0/intermediate/dense/mul/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.5 } } } } node { name: "bert/encoder/layer_0/intermediate/dense/mul" op: "Mul" input: "bert/encoder/layer_0/intermediate/dense/mul/x" input: "bert/encoder/layer_0/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_0/intermediate/dense/mul_1" op: "Mul" input: "bert/encoder/layer_0/intermediate/dense/BiasAdd" input: "bert/encoder/layer_0/intermediate/dense/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/output/dense/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/output/dense/kernel/Assign" op: "Assign" input: "bert/encoder/layer_0/output/dense/kernel" input: "bert/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/output/dense/kernel/read" op: "Identity" input: "bert/encoder/layer_0/output/dense/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/output/dense/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/output/dense/bias/Assign" op: "Assign" input: "bert/encoder/layer_0/output/dense/bias" input: "bert/encoder/layer_0/output/dense/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/output/dense/bias/read" op: "Identity" input: "bert/encoder/layer_0/output/dense/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_0/intermediate/dense/mul_1" input: "bert/encoder/layer_0/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_0/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_0/output/dense/MatMul" input: "bert/encoder/layer_0/output/dense/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_0/output/dropout/keep_prob" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "bert/encoder/layer_0/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_0/output/dropout/random_uniform/min" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/output/dropout/random_uniform/max" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_0/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_0/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_0/output/dropout/random_uniform/sub" op: "Sub" input: "bert/encoder/layer_0/output/dropout/random_uniform/max" input: "bert/encoder/layer_0/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/output/dropout/random_uniform/mul" op: "Mul" input: "bert/encoder/layer_0/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_0/output/dropout/random_uniform/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/output/dropout/random_uniform" op: "Add" input: "bert/encoder/layer_0/output/dropout/random_uniform/mul" input: "bert/encoder/layer_0/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/output/dropout/add" op: "Add" input: "bert/encoder/layer_0/output/dropout/keep_prob" input: "bert/encoder/layer_0/output/dropout/random_uniform" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/output/dropout/Floor" op: "Floor" input: "bert/encoder/layer_0/output/dropout/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/output/dropout/div" op: "RealDiv" input: "bert/encoder/layer_0/output/dense/BiasAdd" input: "bert/encoder/layer_0/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/output/dropout/mul" op: "Mul" input: "bert/encoder/layer_0/output/dropout/div" input: "bert/encoder/layer_0/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/output/add" op: "Add" input: "bert/encoder/layer_0/output/dropout/mul" input: "bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/output/LayerNorm/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/output/LayerNorm/beta" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/output/LayerNorm/beta/Assign" op: "Assign" input: "bert/encoder/layer_0/output/LayerNorm/beta" input: "bert/encoder/layer_0/output/LayerNorm/beta/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/output/LayerNorm/beta/read" op: "Identity" input: "bert/encoder/layer_0/output/LayerNorm/beta" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/output/LayerNorm/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_0/output/LayerNorm/gamma" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/output/LayerNorm/gamma/Assign" op: "Assign" input: "bert/encoder/layer_0/output/LayerNorm/gamma" input: "bert/encoder/layer_0/output/LayerNorm/gamma/Initializer/ones" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/output/LayerNorm/gamma/read" op: "Identity" input: "bert/encoder/layer_0/output/LayerNorm/gamma" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/output/LayerNorm/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_0/output/LayerNorm/moments/mean" op: "Mean" input: "bert/encoder/layer_0/output/add" input: "bert/encoder/layer_0/output/LayerNorm/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_0/output/LayerNorm/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_0/output/LayerNorm/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_0/output/LayerNorm/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_0/output/add" input: "bert/encoder/layer_0/output/LayerNorm/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/output/LayerNorm/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_0/output/LayerNorm/moments/variance" op: "Mean" input: "bert/encoder/layer_0/output/LayerNorm/moments/SquaredDifference" input: "bert/encoder/layer_0/output/LayerNorm/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_0/output/LayerNorm/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_0/output/LayerNorm/batchnorm/add" op: "Add" input: "bert/encoder/layer_0/output/LayerNorm/moments/variance" input: "bert/encoder/layer_0/output/LayerNorm/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_0/output/LayerNorm/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_0/output/LayerNorm/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_0/output/LayerNorm/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_0/output/LayerNorm/batchnorm/Rsqrt" input: "bert/encoder/layer_0/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_0/output/add" input: "bert/encoder/layer_0/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_0/output/LayerNorm/moments/mean" input: "bert/encoder/layer_0/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/output/LayerNorm/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_0/output/LayerNorm/beta/read" input: "bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/output/LayerNorm/batchnorm/add_1" op: "Add" input: "bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_1" input: "bert/encoder/layer_0/output/LayerNorm/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/Assign" op: "Assign" input: "bert/encoder/layer_1/attention/self/query/kernel" input: "bert/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/read" op: "Identity" input: "bert/encoder/layer_1/attention/self/query/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/self/query/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/self/query/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/attention/self/query/bias/Assign" op: "Assign" input: "bert/encoder/layer_1/attention/self/query/bias" input: "bert/encoder/layer_1/attention/self/query/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/attention/self/query/bias/read" op: "Identity" input: "bert/encoder/layer_1/attention/self/query/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/self/query/MatMul" op: "MatMul" input: "bert/encoder/layer_0/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_1/attention/self/query/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_1/attention/self/query/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_1/attention/self/query/MatMul" input: "bert/encoder/layer_1/attention/self/query/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/Assign" op: "Assign" input: "bert/encoder/layer_1/attention/self/key/kernel" input: "bert/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/read" op: "Identity" input: "bert/encoder/layer_1/attention/self/key/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/self/key/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/self/key/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/attention/self/key/bias/Assign" op: "Assign" input: "bert/encoder/layer_1/attention/self/key/bias" input: "bert/encoder/layer_1/attention/self/key/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/attention/self/key/bias/read" op: "Identity" input: "bert/encoder/layer_1/attention/self/key/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/self/key/MatMul" op: "MatMul" input: "bert/encoder/layer_0/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_1/attention/self/key/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_1/attention/self/key/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_1/attention/self/key/MatMul" input: "bert/encoder/layer_1/attention/self/key/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/Assign" op: "Assign" input: "bert/encoder/layer_1/attention/self/value/kernel" input: "bert/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/read" op: "Identity" input: "bert/encoder/layer_1/attention/self/value/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/self/value/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/self/value/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/attention/self/value/bias/Assign" op: "Assign" input: "bert/encoder/layer_1/attention/self/value/bias" input: "bert/encoder/layer_1/attention/self/value/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/attention/self/value/bias/read" op: "Identity" input: "bert/encoder/layer_1/attention/self/value/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/self/value/MatMul" op: "MatMul" input: "bert/encoder/layer_0/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_1/attention/self/value/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_1/attention/self/value/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_1/attention/self/value/MatMul" input: "bert/encoder/layer_1/attention/self/value/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_1/attention/self/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_1/attention/self/Reshape" op: "Reshape" input: "bert/encoder/layer_1/attention/self/query/BiasAdd" input: "bert/encoder/layer_1/attention/self/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_1/attention/self/transpose/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_1/attention/self/transpose" op: "Transpose" input: "bert/encoder/layer_1/attention/self/Reshape" input: "bert/encoder/layer_1/attention/self/transpose/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_1/attention/self/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_1/attention/self/Reshape_1" op: "Reshape" input: "bert/encoder/layer_1/attention/self/key/BiasAdd" input: "bert/encoder/layer_1/attention/self/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_1/attention/self/transpose_1/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_1/attention/self/transpose_1" op: "Transpose" input: "bert/encoder/layer_1/attention/self/Reshape_1" input: "bert/encoder/layer_1/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_1/attention/self/MatMul" op: "BatchMatMul" input: "bert/encoder/layer_1/attention/self/transpose" input: "bert/encoder/layer_1/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "bert/encoder/layer_1/attention/self/Mul/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.125 } } } } node { name: "bert/encoder/layer_1/attention/self/Mul" op: "Mul" input: "bert/encoder/layer_1/attention/self/MatMul" input: "bert/encoder/layer_1/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_1/attention/self/ExpandDims/dim" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_1/attention/self/ExpandDims" op: "ExpandDims" input: "bert/encoder/mul" input: "bert/encoder/layer_1/attention/self/ExpandDims/dim" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tdim" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_1/attention/self/sub/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_1/attention/self/sub" op: "Sub" input: "bert/encoder/layer_1/attention/self/sub/x" input: "bert/encoder/layer_1/attention/self/ExpandDims" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_1/attention/self/mul_1/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: -10000.0 } } } } node { name: "bert/encoder/layer_1/attention/self/mul_1" op: "Mul" input: "bert/encoder/layer_1/attention/self/sub" input: "bert/encoder/layer_1/attention/self/mul_1/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_1/attention/self/add" op: "Add" input: "bert/encoder/layer_1/attention/self/Mul" input: "bert/encoder/layer_1/attention/self/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_1/attention/self/Softmax" op: "Softmax" input: "bert/encoder/layer_1/attention/self/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_1/attention/self/dropout/keep_prob" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "bert/encoder/layer_1/attention/self/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "bert/encoder/layer_1/attention/self/dropout/random_uniform/min" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/self/dropout/random_uniform/max" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_1/attention/self/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_1/attention/self/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_1/attention/self/dropout/random_uniform/sub" op: "Sub" input: "bert/encoder/layer_1/attention/self/dropout/random_uniform/max" input: "bert/encoder/layer_1/attention/self/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/attention/self/dropout/random_uniform/mul" op: "Mul" input: "bert/encoder/layer_1/attention/self/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_1/attention/self/dropout/random_uniform/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_1/attention/self/dropout/random_uniform" op: "Add" input: "bert/encoder/layer_1/attention/self/dropout/random_uniform/mul" input: "bert/encoder/layer_1/attention/self/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_1/attention/self/dropout/add" op: "Add" input: "bert/encoder/layer_1/attention/self/dropout/keep_prob" input: "bert/encoder/layer_1/attention/self/dropout/random_uniform" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_1/attention/self/dropout/Floor" op: "Floor" input: "bert/encoder/layer_1/attention/self/dropout/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_1/attention/self/dropout/div" op: "RealDiv" input: "bert/encoder/layer_1/attention/self/Softmax" input: "bert/encoder/layer_1/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_1/attention/self/dropout/mul" op: "Mul" input: "bert/encoder/layer_1/attention/self/dropout/div" input: "bert/encoder/layer_1/attention/self/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_1/attention/self/Reshape_2/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_1/attention/self/Reshape_2" op: "Reshape" input: "bert/encoder/layer_1/attention/self/value/BiasAdd" input: "bert/encoder/layer_1/attention/self/Reshape_2/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_1/attention/self/transpose_2/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_1/attention/self/transpose_2" op: "Transpose" input: "bert/encoder/layer_1/attention/self/Reshape_2" input: "bert/encoder/layer_1/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_1/attention/self/MatMul_1" op: "BatchMatMul" input: "bert/encoder/layer_1/attention/self/dropout/mul" input: "bert/encoder/layer_1/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "bert/encoder/layer_1/attention/self/transpose_3/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_1/attention/self/transpose_3" op: "Transpose" input: "bert/encoder/layer_1/attention/self/MatMul_1" input: "bert/encoder/layer_1/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_1/attention/self/Reshape_3/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_1/attention/self/Reshape_3" op: "Reshape" input: "bert/encoder/layer_1/attention/self/transpose_3" input: "bert/encoder/layer_1/attention/self/Reshape_3/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/Assign" op: "Assign" input: "bert/encoder/layer_1/attention/output/dense/kernel" input: "bert/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/read" op: "Identity" input: "bert/encoder/layer_1/attention/output/dense/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/output/dense/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/attention/output/dense/bias/Assign" op: "Assign" input: "bert/encoder/layer_1/attention/output/dense/bias" input: "bert/encoder/layer_1/attention/output/dense/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/attention/output/dense/bias/read" op: "Identity" input: "bert/encoder/layer_1/attention/output/dense/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_1/attention/self/Reshape_3" input: "bert/encoder/layer_1/attention/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_1/attention/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_1/attention/output/dense/MatMul" input: "bert/encoder/layer_1/attention/output/dense/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_1/attention/output/dropout/keep_prob" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "bert/encoder/layer_1/attention/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_1/attention/output/dropout/random_uniform/min" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/output/dropout/random_uniform/max" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_1/attention/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_1/attention/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_1/attention/output/dropout/random_uniform/sub" op: "Sub" input: "bert/encoder/layer_1/attention/output/dropout/random_uniform/max" input: "bert/encoder/layer_1/attention/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/attention/output/dropout/random_uniform/mul" op: "Mul" input: "bert/encoder/layer_1/attention/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_1/attention/output/dropout/random_uniform/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/output/dropout/random_uniform" op: "Add" input: "bert/encoder/layer_1/attention/output/dropout/random_uniform/mul" input: "bert/encoder/layer_1/attention/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/output/dropout/add" op: "Add" input: "bert/encoder/layer_1/attention/output/dropout/keep_prob" input: "bert/encoder/layer_1/attention/output/dropout/random_uniform" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/output/dropout/Floor" op: "Floor" input: "bert/encoder/layer_1/attention/output/dropout/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/output/dropout/div" op: "RealDiv" input: "bert/encoder/layer_1/attention/output/dense/BiasAdd" input: "bert/encoder/layer_1/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/output/dropout/mul" op: "Mul" input: "bert/encoder/layer_1/attention/output/dropout/div" input: "bert/encoder/layer_1/attention/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/output/add" op: "Add" input: "bert/encoder/layer_1/attention/output/dropout/mul" input: "bert/encoder/layer_0/output/LayerNorm/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/output/LayerNorm/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/output/LayerNorm/beta" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/attention/output/LayerNorm/beta/Assign" op: "Assign" input: "bert/encoder/layer_1/attention/output/LayerNorm/beta" input: "bert/encoder/layer_1/attention/output/LayerNorm/beta/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/attention/output/LayerNorm/beta/read" op: "Identity" input: "bert/encoder/layer_1/attention/output/LayerNorm/beta" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_1/attention/output/LayerNorm/gamma" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/Assign" op: "Assign" input: "bert/encoder/layer_1/attention/output/LayerNorm/gamma" input: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/Initializer/ones" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/read" op: "Identity" input: "bert/encoder/layer_1/attention/output/LayerNorm/gamma" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/output/LayerNorm/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_1/attention/output/LayerNorm/moments/mean" op: "Mean" input: "bert/encoder/layer_1/attention/output/add" input: "bert/encoder/layer_1/attention/output/LayerNorm/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_1/attention/output/LayerNorm/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_1/attention/output/LayerNorm/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_1/attention/output/add" input: "bert/encoder/layer_1/attention/output/LayerNorm/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/output/LayerNorm/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_1/attention/output/LayerNorm/moments/variance" op: "Mean" input: "bert/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference" input: "bert/encoder/layer_1/attention/output/LayerNorm/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/add" op: "Add" input: "bert/encoder/layer_1/attention/output/LayerNorm/moments/variance" input: "bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/Rsqrt" input: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_1/attention/output/add" input: "bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_1/attention/output/LayerNorm/moments/mean" input: "bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_1/attention/output/LayerNorm/beta/read" input: "bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_1" op: "Add" input: "bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_1" input: "bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/Assign" op: "Assign" input: "bert/encoder/layer_1/intermediate/dense/kernel" input: "bert/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/read" op: "Identity" input: "bert/encoder/layer_1/intermediate/dense/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_1/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_1/intermediate/dense/bias/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/intermediate/dense/bias/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_1/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_1/intermediate/dense/bias/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_1/intermediate/dense/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/intermediate/dense/bias/Assign" op: "Assign" input: "bert/encoder/layer_1/intermediate/dense/bias" input: "bert/encoder/layer_1/intermediate/dense/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/intermediate/dense/bias/read" op: "Identity" input: "bert/encoder/layer_1/intermediate/dense/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_1/intermediate/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_1/intermediate/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_1/intermediate/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_1/intermediate/dense/MatMul" input: "bert/encoder/layer_1/intermediate/dense/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_1/intermediate/dense/Sqrt/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "bert/encoder/layer_1/intermediate/dense/Sqrt" op: "Sqrt" input: "bert/encoder/layer_1/intermediate/dense/Sqrt/x" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/intermediate/dense/truediv" op: "RealDiv" input: "bert/encoder/layer_1/intermediate/dense/BiasAdd" input: "bert/encoder/layer_1/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_1/intermediate/dense/Erf" op: "Erf" input: "bert/encoder/layer_1/intermediate/dense/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_1/intermediate/dense/add/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_1/intermediate/dense/add" op: "Add" input: "bert/encoder/layer_1/intermediate/dense/add/x" input: "bert/encoder/layer_1/intermediate/dense/Erf" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_1/intermediate/dense/mul/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.5 } } } } node { name: "bert/encoder/layer_1/intermediate/dense/mul" op: "Mul" input: "bert/encoder/layer_1/intermediate/dense/mul/x" input: "bert/encoder/layer_1/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_1/intermediate/dense/mul_1" op: "Mul" input: "bert/encoder/layer_1/intermediate/dense/BiasAdd" input: "bert/encoder/layer_1/intermediate/dense/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/output/dense/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/output/dense/kernel/Assign" op: "Assign" input: "bert/encoder/layer_1/output/dense/kernel" input: "bert/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/output/dense/kernel/read" op: "Identity" input: "bert/encoder/layer_1/output/dense/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/output/dense/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/output/dense/bias/Assign" op: "Assign" input: "bert/encoder/layer_1/output/dense/bias" input: "bert/encoder/layer_1/output/dense/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/output/dense/bias/read" op: "Identity" input: "bert/encoder/layer_1/output/dense/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_1/intermediate/dense/mul_1" input: "bert/encoder/layer_1/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_1/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_1/output/dense/MatMul" input: "bert/encoder/layer_1/output/dense/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_1/output/dropout/keep_prob" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "bert/encoder/layer_1/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_1/output/dropout/random_uniform/min" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/output/dropout/random_uniform/max" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_1/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_1/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_1/output/dropout/random_uniform/sub" op: "Sub" input: "bert/encoder/layer_1/output/dropout/random_uniform/max" input: "bert/encoder/layer_1/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/output/dropout/random_uniform/mul" op: "Mul" input: "bert/encoder/layer_1/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_1/output/dropout/random_uniform/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/output/dropout/random_uniform" op: "Add" input: "bert/encoder/layer_1/output/dropout/random_uniform/mul" input: "bert/encoder/layer_1/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/output/dropout/add" op: "Add" input: "bert/encoder/layer_1/output/dropout/keep_prob" input: "bert/encoder/layer_1/output/dropout/random_uniform" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/output/dropout/Floor" op: "Floor" input: "bert/encoder/layer_1/output/dropout/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/output/dropout/div" op: "RealDiv" input: "bert/encoder/layer_1/output/dense/BiasAdd" input: "bert/encoder/layer_1/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/output/dropout/mul" op: "Mul" input: "bert/encoder/layer_1/output/dropout/div" input: "bert/encoder/layer_1/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/output/add" op: "Add" input: "bert/encoder/layer_1/output/dropout/mul" input: "bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/output/LayerNorm/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/output/LayerNorm/beta" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/output/LayerNorm/beta/Assign" op: "Assign" input: "bert/encoder/layer_1/output/LayerNorm/beta" input: "bert/encoder/layer_1/output/LayerNorm/beta/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/output/LayerNorm/beta/read" op: "Identity" input: "bert/encoder/layer_1/output/LayerNorm/beta" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/output/LayerNorm/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_1/output/LayerNorm/gamma" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/output/LayerNorm/gamma/Assign" op: "Assign" input: "bert/encoder/layer_1/output/LayerNorm/gamma" input: "bert/encoder/layer_1/output/LayerNorm/gamma/Initializer/ones" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/output/LayerNorm/gamma/read" op: "Identity" input: "bert/encoder/layer_1/output/LayerNorm/gamma" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/output/LayerNorm/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_1/output/LayerNorm/moments/mean" op: "Mean" input: "bert/encoder/layer_1/output/add" input: "bert/encoder/layer_1/output/LayerNorm/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_1/output/LayerNorm/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_1/output/LayerNorm/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_1/output/LayerNorm/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_1/output/add" input: "bert/encoder/layer_1/output/LayerNorm/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/output/LayerNorm/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_1/output/LayerNorm/moments/variance" op: "Mean" input: "bert/encoder/layer_1/output/LayerNorm/moments/SquaredDifference" input: "bert/encoder/layer_1/output/LayerNorm/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_1/output/LayerNorm/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_1/output/LayerNorm/batchnorm/add" op: "Add" input: "bert/encoder/layer_1/output/LayerNorm/moments/variance" input: "bert/encoder/layer_1/output/LayerNorm/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_1/output/LayerNorm/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_1/output/LayerNorm/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_1/output/LayerNorm/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_1/output/LayerNorm/batchnorm/Rsqrt" input: "bert/encoder/layer_1/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_1/output/add" input: "bert/encoder/layer_1/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_1/output/LayerNorm/moments/mean" input: "bert/encoder/layer_1/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/output/LayerNorm/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_1/output/LayerNorm/beta/read" input: "bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/output/LayerNorm/batchnorm/add_1" op: "Add" input: "bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_1" input: "bert/encoder/layer_1/output/LayerNorm/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/Assign" op: "Assign" input: "bert/encoder/layer_2/attention/self/query/kernel" input: "bert/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/read" op: "Identity" input: "bert/encoder/layer_2/attention/self/query/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/self/query/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/self/query/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/attention/self/query/bias/Assign" op: "Assign" input: "bert/encoder/layer_2/attention/self/query/bias" input: "bert/encoder/layer_2/attention/self/query/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/attention/self/query/bias/read" op: "Identity" input: "bert/encoder/layer_2/attention/self/query/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/self/query/MatMul" op: "MatMul" input: "bert/encoder/layer_1/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_2/attention/self/query/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_2/attention/self/query/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_2/attention/self/query/MatMul" input: "bert/encoder/layer_2/attention/self/query/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/Assign" op: "Assign" input: "bert/encoder/layer_2/attention/self/key/kernel" input: "bert/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/read" op: "Identity" input: "bert/encoder/layer_2/attention/self/key/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/self/key/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/self/key/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/attention/self/key/bias/Assign" op: "Assign" input: "bert/encoder/layer_2/attention/self/key/bias" input: "bert/encoder/layer_2/attention/self/key/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/attention/self/key/bias/read" op: "Identity" input: "bert/encoder/layer_2/attention/self/key/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/self/key/MatMul" op: "MatMul" input: "bert/encoder/layer_1/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_2/attention/self/key/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_2/attention/self/key/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_2/attention/self/key/MatMul" input: "bert/encoder/layer_2/attention/self/key/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/Assign" op: "Assign" input: "bert/encoder/layer_2/attention/self/value/kernel" input: "bert/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/read" op: "Identity" input: "bert/encoder/layer_2/attention/self/value/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/self/value/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/self/value/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/attention/self/value/bias/Assign" op: "Assign" input: "bert/encoder/layer_2/attention/self/value/bias" input: "bert/encoder/layer_2/attention/self/value/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/attention/self/value/bias/read" op: "Identity" input: "bert/encoder/layer_2/attention/self/value/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/self/value/MatMul" op: "MatMul" input: "bert/encoder/layer_1/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_2/attention/self/value/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_2/attention/self/value/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_2/attention/self/value/MatMul" input: "bert/encoder/layer_2/attention/self/value/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_2/attention/self/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_2/attention/self/Reshape" op: "Reshape" input: "bert/encoder/layer_2/attention/self/query/BiasAdd" input: "bert/encoder/layer_2/attention/self/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_2/attention/self/transpose/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_2/attention/self/transpose" op: "Transpose" input: "bert/encoder/layer_2/attention/self/Reshape" input: "bert/encoder/layer_2/attention/self/transpose/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_2/attention/self/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_2/attention/self/Reshape_1" op: "Reshape" input: "bert/encoder/layer_2/attention/self/key/BiasAdd" input: "bert/encoder/layer_2/attention/self/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_2/attention/self/transpose_1/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_2/attention/self/transpose_1" op: "Transpose" input: "bert/encoder/layer_2/attention/self/Reshape_1" input: "bert/encoder/layer_2/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_2/attention/self/MatMul" op: "BatchMatMul" input: "bert/encoder/layer_2/attention/self/transpose" input: "bert/encoder/layer_2/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "bert/encoder/layer_2/attention/self/Mul/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.125 } } } } node { name: "bert/encoder/layer_2/attention/self/Mul" op: "Mul" input: "bert/encoder/layer_2/attention/self/MatMul" input: "bert/encoder/layer_2/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_2/attention/self/ExpandDims/dim" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_2/attention/self/ExpandDims" op: "ExpandDims" input: "bert/encoder/mul" input: "bert/encoder/layer_2/attention/self/ExpandDims/dim" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tdim" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_2/attention/self/sub/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_2/attention/self/sub" op: "Sub" input: "bert/encoder/layer_2/attention/self/sub/x" input: "bert/encoder/layer_2/attention/self/ExpandDims" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_2/attention/self/mul_1/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: -10000.0 } } } } node { name: "bert/encoder/layer_2/attention/self/mul_1" op: "Mul" input: "bert/encoder/layer_2/attention/self/sub" input: "bert/encoder/layer_2/attention/self/mul_1/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_2/attention/self/add" op: "Add" input: "bert/encoder/layer_2/attention/self/Mul" input: "bert/encoder/layer_2/attention/self/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_2/attention/self/Softmax" op: "Softmax" input: "bert/encoder/layer_2/attention/self/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_2/attention/self/dropout/keep_prob" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "bert/encoder/layer_2/attention/self/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "bert/encoder/layer_2/attention/self/dropout/random_uniform/min" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/self/dropout/random_uniform/max" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_2/attention/self/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_2/attention/self/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_2/attention/self/dropout/random_uniform/sub" op: "Sub" input: "bert/encoder/layer_2/attention/self/dropout/random_uniform/max" input: "bert/encoder/layer_2/attention/self/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/attention/self/dropout/random_uniform/mul" op: "Mul" input: "bert/encoder/layer_2/attention/self/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_2/attention/self/dropout/random_uniform/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_2/attention/self/dropout/random_uniform" op: "Add" input: "bert/encoder/layer_2/attention/self/dropout/random_uniform/mul" input: "bert/encoder/layer_2/attention/self/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_2/attention/self/dropout/add" op: "Add" input: "bert/encoder/layer_2/attention/self/dropout/keep_prob" input: "bert/encoder/layer_2/attention/self/dropout/random_uniform" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_2/attention/self/dropout/Floor" op: "Floor" input: "bert/encoder/layer_2/attention/self/dropout/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_2/attention/self/dropout/div" op: "RealDiv" input: "bert/encoder/layer_2/attention/self/Softmax" input: "bert/encoder/layer_2/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_2/attention/self/dropout/mul" op: "Mul" input: "bert/encoder/layer_2/attention/self/dropout/div" input: "bert/encoder/layer_2/attention/self/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_2/attention/self/Reshape_2/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_2/attention/self/Reshape_2" op: "Reshape" input: "bert/encoder/layer_2/attention/self/value/BiasAdd" input: "bert/encoder/layer_2/attention/self/Reshape_2/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_2/attention/self/transpose_2/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_2/attention/self/transpose_2" op: "Transpose" input: "bert/encoder/layer_2/attention/self/Reshape_2" input: "bert/encoder/layer_2/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_2/attention/self/MatMul_1" op: "BatchMatMul" input: "bert/encoder/layer_2/attention/self/dropout/mul" input: "bert/encoder/layer_2/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "bert/encoder/layer_2/attention/self/transpose_3/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_2/attention/self/transpose_3" op: "Transpose" input: "bert/encoder/layer_2/attention/self/MatMul_1" input: "bert/encoder/layer_2/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_2/attention/self/Reshape_3/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_2/attention/self/Reshape_3" op: "Reshape" input: "bert/encoder/layer_2/attention/self/transpose_3" input: "bert/encoder/layer_2/attention/self/Reshape_3/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/Assign" op: "Assign" input: "bert/encoder/layer_2/attention/output/dense/kernel" input: "bert/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/read" op: "Identity" input: "bert/encoder/layer_2/attention/output/dense/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/output/dense/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/attention/output/dense/bias/Assign" op: "Assign" input: "bert/encoder/layer_2/attention/output/dense/bias" input: "bert/encoder/layer_2/attention/output/dense/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/attention/output/dense/bias/read" op: "Identity" input: "bert/encoder/layer_2/attention/output/dense/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_2/attention/self/Reshape_3" input: "bert/encoder/layer_2/attention/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_2/attention/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_2/attention/output/dense/MatMul" input: "bert/encoder/layer_2/attention/output/dense/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_2/attention/output/dropout/keep_prob" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "bert/encoder/layer_2/attention/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_2/attention/output/dropout/random_uniform/min" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/output/dropout/random_uniform/max" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_2/attention/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_2/attention/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_2/attention/output/dropout/random_uniform/sub" op: "Sub" input: "bert/encoder/layer_2/attention/output/dropout/random_uniform/max" input: "bert/encoder/layer_2/attention/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/attention/output/dropout/random_uniform/mul" op: "Mul" input: "bert/encoder/layer_2/attention/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_2/attention/output/dropout/random_uniform/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/output/dropout/random_uniform" op: "Add" input: "bert/encoder/layer_2/attention/output/dropout/random_uniform/mul" input: "bert/encoder/layer_2/attention/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/output/dropout/add" op: "Add" input: "bert/encoder/layer_2/attention/output/dropout/keep_prob" input: "bert/encoder/layer_2/attention/output/dropout/random_uniform" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/output/dropout/Floor" op: "Floor" input: "bert/encoder/layer_2/attention/output/dropout/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/output/dropout/div" op: "RealDiv" input: "bert/encoder/layer_2/attention/output/dense/BiasAdd" input: "bert/encoder/layer_2/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/output/dropout/mul" op: "Mul" input: "bert/encoder/layer_2/attention/output/dropout/div" input: "bert/encoder/layer_2/attention/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/output/add" op: "Add" input: "bert/encoder/layer_2/attention/output/dropout/mul" input: "bert/encoder/layer_1/output/LayerNorm/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/output/LayerNorm/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/output/LayerNorm/beta" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/attention/output/LayerNorm/beta/Assign" op: "Assign" input: "bert/encoder/layer_2/attention/output/LayerNorm/beta" input: "bert/encoder/layer_2/attention/output/LayerNorm/beta/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/attention/output/LayerNorm/beta/read" op: "Identity" input: "bert/encoder/layer_2/attention/output/LayerNorm/beta" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_2/attention/output/LayerNorm/gamma" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/Assign" op: "Assign" input: "bert/encoder/layer_2/attention/output/LayerNorm/gamma" input: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/Initializer/ones" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/read" op: "Identity" input: "bert/encoder/layer_2/attention/output/LayerNorm/gamma" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/output/LayerNorm/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_2/attention/output/LayerNorm/moments/mean" op: "Mean" input: "bert/encoder/layer_2/attention/output/add" input: "bert/encoder/layer_2/attention/output/LayerNorm/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_2/attention/output/LayerNorm/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_2/attention/output/LayerNorm/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_2/attention/output/add" input: "bert/encoder/layer_2/attention/output/LayerNorm/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/output/LayerNorm/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_2/attention/output/LayerNorm/moments/variance" op: "Mean" input: "bert/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference" input: "bert/encoder/layer_2/attention/output/LayerNorm/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/add" op: "Add" input: "bert/encoder/layer_2/attention/output/LayerNorm/moments/variance" input: "bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/Rsqrt" input: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_2/attention/output/add" input: "bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_2/attention/output/LayerNorm/moments/mean" input: "bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_2/attention/output/LayerNorm/beta/read" input: "bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_1" op: "Add" input: "bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_1" input: "bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/Assign" op: "Assign" input: "bert/encoder/layer_2/intermediate/dense/kernel" input: "bert/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/read" op: "Identity" input: "bert/encoder/layer_2/intermediate/dense/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_2/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_2/intermediate/dense/bias/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/intermediate/dense/bias/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_2/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_2/intermediate/dense/bias/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_2/intermediate/dense/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/intermediate/dense/bias/Assign" op: "Assign" input: "bert/encoder/layer_2/intermediate/dense/bias" input: "bert/encoder/layer_2/intermediate/dense/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/intermediate/dense/bias/read" op: "Identity" input: "bert/encoder/layer_2/intermediate/dense/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_2/intermediate/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_2/intermediate/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_2/intermediate/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_2/intermediate/dense/MatMul" input: "bert/encoder/layer_2/intermediate/dense/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_2/intermediate/dense/Sqrt/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "bert/encoder/layer_2/intermediate/dense/Sqrt" op: "Sqrt" input: "bert/encoder/layer_2/intermediate/dense/Sqrt/x" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/intermediate/dense/truediv" op: "RealDiv" input: "bert/encoder/layer_2/intermediate/dense/BiasAdd" input: "bert/encoder/layer_2/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_2/intermediate/dense/Erf" op: "Erf" input: "bert/encoder/layer_2/intermediate/dense/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_2/intermediate/dense/add/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_2/intermediate/dense/add" op: "Add" input: "bert/encoder/layer_2/intermediate/dense/add/x" input: "bert/encoder/layer_2/intermediate/dense/Erf" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_2/intermediate/dense/mul/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.5 } } } } node { name: "bert/encoder/layer_2/intermediate/dense/mul" op: "Mul" input: "bert/encoder/layer_2/intermediate/dense/mul/x" input: "bert/encoder/layer_2/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_2/intermediate/dense/mul_1" op: "Mul" input: "bert/encoder/layer_2/intermediate/dense/BiasAdd" input: "bert/encoder/layer_2/intermediate/dense/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/output/dense/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/output/dense/kernel/Assign" op: "Assign" input: "bert/encoder/layer_2/output/dense/kernel" input: "bert/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/output/dense/kernel/read" op: "Identity" input: "bert/encoder/layer_2/output/dense/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/output/dense/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/output/dense/bias/Assign" op: "Assign" input: "bert/encoder/layer_2/output/dense/bias" input: "bert/encoder/layer_2/output/dense/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/output/dense/bias/read" op: "Identity" input: "bert/encoder/layer_2/output/dense/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_2/intermediate/dense/mul_1" input: "bert/encoder/layer_2/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_2/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_2/output/dense/MatMul" input: "bert/encoder/layer_2/output/dense/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_2/output/dropout/keep_prob" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "bert/encoder/layer_2/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_2/output/dropout/random_uniform/min" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/output/dropout/random_uniform/max" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_2/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_2/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_2/output/dropout/random_uniform/sub" op: "Sub" input: "bert/encoder/layer_2/output/dropout/random_uniform/max" input: "bert/encoder/layer_2/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/output/dropout/random_uniform/mul" op: "Mul" input: "bert/encoder/layer_2/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_2/output/dropout/random_uniform/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/output/dropout/random_uniform" op: "Add" input: "bert/encoder/layer_2/output/dropout/random_uniform/mul" input: "bert/encoder/layer_2/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/output/dropout/add" op: "Add" input: "bert/encoder/layer_2/output/dropout/keep_prob" input: "bert/encoder/layer_2/output/dropout/random_uniform" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/output/dropout/Floor" op: "Floor" input: "bert/encoder/layer_2/output/dropout/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/output/dropout/div" op: "RealDiv" input: "bert/encoder/layer_2/output/dense/BiasAdd" input: "bert/encoder/layer_2/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/output/dropout/mul" op: "Mul" input: "bert/encoder/layer_2/output/dropout/div" input: "bert/encoder/layer_2/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/output/add" op: "Add" input: "bert/encoder/layer_2/output/dropout/mul" input: "bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/output/LayerNorm/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/output/LayerNorm/beta" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/output/LayerNorm/beta/Assign" op: "Assign" input: "bert/encoder/layer_2/output/LayerNorm/beta" input: "bert/encoder/layer_2/output/LayerNorm/beta/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/output/LayerNorm/beta/read" op: "Identity" input: "bert/encoder/layer_2/output/LayerNorm/beta" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/output/LayerNorm/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_2/output/LayerNorm/gamma" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/output/LayerNorm/gamma/Assign" op: "Assign" input: "bert/encoder/layer_2/output/LayerNorm/gamma" input: "bert/encoder/layer_2/output/LayerNorm/gamma/Initializer/ones" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/output/LayerNorm/gamma/read" op: "Identity" input: "bert/encoder/layer_2/output/LayerNorm/gamma" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/output/LayerNorm/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_2/output/LayerNorm/moments/mean" op: "Mean" input: "bert/encoder/layer_2/output/add" input: "bert/encoder/layer_2/output/LayerNorm/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_2/output/LayerNorm/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_2/output/LayerNorm/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_2/output/LayerNorm/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_2/output/add" input: "bert/encoder/layer_2/output/LayerNorm/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/output/LayerNorm/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_2/output/LayerNorm/moments/variance" op: "Mean" input: "bert/encoder/layer_2/output/LayerNorm/moments/SquaredDifference" input: "bert/encoder/layer_2/output/LayerNorm/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_2/output/LayerNorm/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_2/output/LayerNorm/batchnorm/add" op: "Add" input: "bert/encoder/layer_2/output/LayerNorm/moments/variance" input: "bert/encoder/layer_2/output/LayerNorm/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_2/output/LayerNorm/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_2/output/LayerNorm/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_2/output/LayerNorm/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_2/output/LayerNorm/batchnorm/Rsqrt" input: "bert/encoder/layer_2/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_2/output/add" input: "bert/encoder/layer_2/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_2/output/LayerNorm/moments/mean" input: "bert/encoder/layer_2/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/output/LayerNorm/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_2/output/LayerNorm/beta/read" input: "bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/output/LayerNorm/batchnorm/add_1" op: "Add" input: "bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_1" input: "bert/encoder/layer_2/output/LayerNorm/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/Assign" op: "Assign" input: "bert/encoder/layer_3/attention/self/query/kernel" input: "bert/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/read" op: "Identity" input: "bert/encoder/layer_3/attention/self/query/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/self/query/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/self/query/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/attention/self/query/bias/Assign" op: "Assign" input: "bert/encoder/layer_3/attention/self/query/bias" input: "bert/encoder/layer_3/attention/self/query/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/attention/self/query/bias/read" op: "Identity" input: "bert/encoder/layer_3/attention/self/query/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/self/query/MatMul" op: "MatMul" input: "bert/encoder/layer_2/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_3/attention/self/query/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_3/attention/self/query/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_3/attention/self/query/MatMul" input: "bert/encoder/layer_3/attention/self/query/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/Assign" op: "Assign" input: "bert/encoder/layer_3/attention/self/key/kernel" input: "bert/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/read" op: "Identity" input: "bert/encoder/layer_3/attention/self/key/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/self/key/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/self/key/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/attention/self/key/bias/Assign" op: "Assign" input: "bert/encoder/layer_3/attention/self/key/bias" input: "bert/encoder/layer_3/attention/self/key/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/attention/self/key/bias/read" op: "Identity" input: "bert/encoder/layer_3/attention/self/key/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/self/key/MatMul" op: "MatMul" input: "bert/encoder/layer_2/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_3/attention/self/key/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_3/attention/self/key/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_3/attention/self/key/MatMul" input: "bert/encoder/layer_3/attention/self/key/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/Assign" op: "Assign" input: "bert/encoder/layer_3/attention/self/value/kernel" input: "bert/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/read" op: "Identity" input: "bert/encoder/layer_3/attention/self/value/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/self/value/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/self/value/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/attention/self/value/bias/Assign" op: "Assign" input: "bert/encoder/layer_3/attention/self/value/bias" input: "bert/encoder/layer_3/attention/self/value/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/attention/self/value/bias/read" op: "Identity" input: "bert/encoder/layer_3/attention/self/value/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/self/value/MatMul" op: "MatMul" input: "bert/encoder/layer_2/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_3/attention/self/value/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_3/attention/self/value/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_3/attention/self/value/MatMul" input: "bert/encoder/layer_3/attention/self/value/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_3/attention/self/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_3/attention/self/Reshape" op: "Reshape" input: "bert/encoder/layer_3/attention/self/query/BiasAdd" input: "bert/encoder/layer_3/attention/self/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_3/attention/self/transpose/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_3/attention/self/transpose" op: "Transpose" input: "bert/encoder/layer_3/attention/self/Reshape" input: "bert/encoder/layer_3/attention/self/transpose/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_3/attention/self/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_3/attention/self/Reshape_1" op: "Reshape" input: "bert/encoder/layer_3/attention/self/key/BiasAdd" input: "bert/encoder/layer_3/attention/self/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_3/attention/self/transpose_1/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_3/attention/self/transpose_1" op: "Transpose" input: "bert/encoder/layer_3/attention/self/Reshape_1" input: "bert/encoder/layer_3/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_3/attention/self/MatMul" op: "BatchMatMul" input: "bert/encoder/layer_3/attention/self/transpose" input: "bert/encoder/layer_3/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "bert/encoder/layer_3/attention/self/Mul/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.125 } } } } node { name: "bert/encoder/layer_3/attention/self/Mul" op: "Mul" input: "bert/encoder/layer_3/attention/self/MatMul" input: "bert/encoder/layer_3/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_3/attention/self/ExpandDims/dim" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_3/attention/self/ExpandDims" op: "ExpandDims" input: "bert/encoder/mul" input: "bert/encoder/layer_3/attention/self/ExpandDims/dim" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tdim" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_3/attention/self/sub/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_3/attention/self/sub" op: "Sub" input: "bert/encoder/layer_3/attention/self/sub/x" input: "bert/encoder/layer_3/attention/self/ExpandDims" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_3/attention/self/mul_1/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: -10000.0 } } } } node { name: "bert/encoder/layer_3/attention/self/mul_1" op: "Mul" input: "bert/encoder/layer_3/attention/self/sub" input: "bert/encoder/layer_3/attention/self/mul_1/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_3/attention/self/add" op: "Add" input: "bert/encoder/layer_3/attention/self/Mul" input: "bert/encoder/layer_3/attention/self/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_3/attention/self/Softmax" op: "Softmax" input: "bert/encoder/layer_3/attention/self/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_3/attention/self/dropout/keep_prob" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "bert/encoder/layer_3/attention/self/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "bert/encoder/layer_3/attention/self/dropout/random_uniform/min" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/self/dropout/random_uniform/max" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_3/attention/self/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_3/attention/self/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_3/attention/self/dropout/random_uniform/sub" op: "Sub" input: "bert/encoder/layer_3/attention/self/dropout/random_uniform/max" input: "bert/encoder/layer_3/attention/self/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/attention/self/dropout/random_uniform/mul" op: "Mul" input: "bert/encoder/layer_3/attention/self/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_3/attention/self/dropout/random_uniform/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_3/attention/self/dropout/random_uniform" op: "Add" input: "bert/encoder/layer_3/attention/self/dropout/random_uniform/mul" input: "bert/encoder/layer_3/attention/self/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_3/attention/self/dropout/add" op: "Add" input: "bert/encoder/layer_3/attention/self/dropout/keep_prob" input: "bert/encoder/layer_3/attention/self/dropout/random_uniform" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_3/attention/self/dropout/Floor" op: "Floor" input: "bert/encoder/layer_3/attention/self/dropout/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_3/attention/self/dropout/div" op: "RealDiv" input: "bert/encoder/layer_3/attention/self/Softmax" input: "bert/encoder/layer_3/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_3/attention/self/dropout/mul" op: "Mul" input: "bert/encoder/layer_3/attention/self/dropout/div" input: "bert/encoder/layer_3/attention/self/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_3/attention/self/Reshape_2/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_3/attention/self/Reshape_2" op: "Reshape" input: "bert/encoder/layer_3/attention/self/value/BiasAdd" input: "bert/encoder/layer_3/attention/self/Reshape_2/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_3/attention/self/transpose_2/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_3/attention/self/transpose_2" op: "Transpose" input: "bert/encoder/layer_3/attention/self/Reshape_2" input: "bert/encoder/layer_3/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_3/attention/self/MatMul_1" op: "BatchMatMul" input: "bert/encoder/layer_3/attention/self/dropout/mul" input: "bert/encoder/layer_3/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "bert/encoder/layer_3/attention/self/transpose_3/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_3/attention/self/transpose_3" op: "Transpose" input: "bert/encoder/layer_3/attention/self/MatMul_1" input: "bert/encoder/layer_3/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_3/attention/self/Reshape_3/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_3/attention/self/Reshape_3" op: "Reshape" input: "bert/encoder/layer_3/attention/self/transpose_3" input: "bert/encoder/layer_3/attention/self/Reshape_3/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/Assign" op: "Assign" input: "bert/encoder/layer_3/attention/output/dense/kernel" input: "bert/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/read" op: "Identity" input: "bert/encoder/layer_3/attention/output/dense/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/output/dense/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/attention/output/dense/bias/Assign" op: "Assign" input: "bert/encoder/layer_3/attention/output/dense/bias" input: "bert/encoder/layer_3/attention/output/dense/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/attention/output/dense/bias/read" op: "Identity" input: "bert/encoder/layer_3/attention/output/dense/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_3/attention/self/Reshape_3" input: "bert/encoder/layer_3/attention/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_3/attention/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_3/attention/output/dense/MatMul" input: "bert/encoder/layer_3/attention/output/dense/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_3/attention/output/dropout/keep_prob" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "bert/encoder/layer_3/attention/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_3/attention/output/dropout/random_uniform/min" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/output/dropout/random_uniform/max" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_3/attention/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_3/attention/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_3/attention/output/dropout/random_uniform/sub" op: "Sub" input: "bert/encoder/layer_3/attention/output/dropout/random_uniform/max" input: "bert/encoder/layer_3/attention/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/attention/output/dropout/random_uniform/mul" op: "Mul" input: "bert/encoder/layer_3/attention/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_3/attention/output/dropout/random_uniform/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/output/dropout/random_uniform" op: "Add" input: "bert/encoder/layer_3/attention/output/dropout/random_uniform/mul" input: "bert/encoder/layer_3/attention/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/output/dropout/add" op: "Add" input: "bert/encoder/layer_3/attention/output/dropout/keep_prob" input: "bert/encoder/layer_3/attention/output/dropout/random_uniform" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/output/dropout/Floor" op: "Floor" input: "bert/encoder/layer_3/attention/output/dropout/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/output/dropout/div" op: "RealDiv" input: "bert/encoder/layer_3/attention/output/dense/BiasAdd" input: "bert/encoder/layer_3/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/output/dropout/mul" op: "Mul" input: "bert/encoder/layer_3/attention/output/dropout/div" input: "bert/encoder/layer_3/attention/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/output/add" op: "Add" input: "bert/encoder/layer_3/attention/output/dropout/mul" input: "bert/encoder/layer_2/output/LayerNorm/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/output/LayerNorm/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/output/LayerNorm/beta" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/attention/output/LayerNorm/beta/Assign" op: "Assign" input: "bert/encoder/layer_3/attention/output/LayerNorm/beta" input: "bert/encoder/layer_3/attention/output/LayerNorm/beta/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/attention/output/LayerNorm/beta/read" op: "Identity" input: "bert/encoder/layer_3/attention/output/LayerNorm/beta" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_3/attention/output/LayerNorm/gamma" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/Assign" op: "Assign" input: "bert/encoder/layer_3/attention/output/LayerNorm/gamma" input: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/Initializer/ones" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/read" op: "Identity" input: "bert/encoder/layer_3/attention/output/LayerNorm/gamma" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/output/LayerNorm/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_3/attention/output/LayerNorm/moments/mean" op: "Mean" input: "bert/encoder/layer_3/attention/output/add" input: "bert/encoder/layer_3/attention/output/LayerNorm/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_3/attention/output/LayerNorm/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_3/attention/output/LayerNorm/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_3/attention/output/add" input: "bert/encoder/layer_3/attention/output/LayerNorm/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/output/LayerNorm/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_3/attention/output/LayerNorm/moments/variance" op: "Mean" input: "bert/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference" input: "bert/encoder/layer_3/attention/output/LayerNorm/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/add" op: "Add" input: "bert/encoder/layer_3/attention/output/LayerNorm/moments/variance" input: "bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/Rsqrt" input: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_3/attention/output/add" input: "bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_3/attention/output/LayerNorm/moments/mean" input: "bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_3/attention/output/LayerNorm/beta/read" input: "bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_1" op: "Add" input: "bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_1" input: "bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/Assign" op: "Assign" input: "bert/encoder/layer_3/intermediate/dense/kernel" input: "bert/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/read" op: "Identity" input: "bert/encoder/layer_3/intermediate/dense/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_3/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_3/intermediate/dense/bias/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/intermediate/dense/bias/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_3/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_3/intermediate/dense/bias/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_3/intermediate/dense/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/intermediate/dense/bias/Assign" op: "Assign" input: "bert/encoder/layer_3/intermediate/dense/bias" input: "bert/encoder/layer_3/intermediate/dense/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/intermediate/dense/bias/read" op: "Identity" input: "bert/encoder/layer_3/intermediate/dense/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_3/intermediate/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_3/intermediate/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_3/intermediate/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_3/intermediate/dense/MatMul" input: "bert/encoder/layer_3/intermediate/dense/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_3/intermediate/dense/Sqrt/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "bert/encoder/layer_3/intermediate/dense/Sqrt" op: "Sqrt" input: "bert/encoder/layer_3/intermediate/dense/Sqrt/x" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/intermediate/dense/truediv" op: "RealDiv" input: "bert/encoder/layer_3/intermediate/dense/BiasAdd" input: "bert/encoder/layer_3/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_3/intermediate/dense/Erf" op: "Erf" input: "bert/encoder/layer_3/intermediate/dense/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_3/intermediate/dense/add/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_3/intermediate/dense/add" op: "Add" input: "bert/encoder/layer_3/intermediate/dense/add/x" input: "bert/encoder/layer_3/intermediate/dense/Erf" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_3/intermediate/dense/mul/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.5 } } } } node { name: "bert/encoder/layer_3/intermediate/dense/mul" op: "Mul" input: "bert/encoder/layer_3/intermediate/dense/mul/x" input: "bert/encoder/layer_3/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_3/intermediate/dense/mul_1" op: "Mul" input: "bert/encoder/layer_3/intermediate/dense/BiasAdd" input: "bert/encoder/layer_3/intermediate/dense/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/output/dense/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/output/dense/kernel/Assign" op: "Assign" input: "bert/encoder/layer_3/output/dense/kernel" input: "bert/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/output/dense/kernel/read" op: "Identity" input: "bert/encoder/layer_3/output/dense/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/output/dense/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/output/dense/bias/Assign" op: "Assign" input: "bert/encoder/layer_3/output/dense/bias" input: "bert/encoder/layer_3/output/dense/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/output/dense/bias/read" op: "Identity" input: "bert/encoder/layer_3/output/dense/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_3/intermediate/dense/mul_1" input: "bert/encoder/layer_3/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_3/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_3/output/dense/MatMul" input: "bert/encoder/layer_3/output/dense/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_3/output/dropout/keep_prob" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "bert/encoder/layer_3/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_3/output/dropout/random_uniform/min" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/output/dropout/random_uniform/max" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_3/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_3/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_3/output/dropout/random_uniform/sub" op: "Sub" input: "bert/encoder/layer_3/output/dropout/random_uniform/max" input: "bert/encoder/layer_3/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/output/dropout/random_uniform/mul" op: "Mul" input: "bert/encoder/layer_3/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_3/output/dropout/random_uniform/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/output/dropout/random_uniform" op: "Add" input: "bert/encoder/layer_3/output/dropout/random_uniform/mul" input: "bert/encoder/layer_3/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/output/dropout/add" op: "Add" input: "bert/encoder/layer_3/output/dropout/keep_prob" input: "bert/encoder/layer_3/output/dropout/random_uniform" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/output/dropout/Floor" op: "Floor" input: "bert/encoder/layer_3/output/dropout/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/output/dropout/div" op: "RealDiv" input: "bert/encoder/layer_3/output/dense/BiasAdd" input: "bert/encoder/layer_3/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/output/dropout/mul" op: "Mul" input: "bert/encoder/layer_3/output/dropout/div" input: "bert/encoder/layer_3/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/output/add" op: "Add" input: "bert/encoder/layer_3/output/dropout/mul" input: "bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/output/LayerNorm/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/output/LayerNorm/beta" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/output/LayerNorm/beta/Assign" op: "Assign" input: "bert/encoder/layer_3/output/LayerNorm/beta" input: "bert/encoder/layer_3/output/LayerNorm/beta/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/output/LayerNorm/beta/read" op: "Identity" input: "bert/encoder/layer_3/output/LayerNorm/beta" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/output/LayerNorm/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_3/output/LayerNorm/gamma" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/output/LayerNorm/gamma/Assign" op: "Assign" input: "bert/encoder/layer_3/output/LayerNorm/gamma" input: "bert/encoder/layer_3/output/LayerNorm/gamma/Initializer/ones" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/output/LayerNorm/gamma/read" op: "Identity" input: "bert/encoder/layer_3/output/LayerNorm/gamma" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/output/LayerNorm/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_3/output/LayerNorm/moments/mean" op: "Mean" input: "bert/encoder/layer_3/output/add" input: "bert/encoder/layer_3/output/LayerNorm/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_3/output/LayerNorm/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_3/output/LayerNorm/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_3/output/LayerNorm/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_3/output/add" input: "bert/encoder/layer_3/output/LayerNorm/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/output/LayerNorm/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_3/output/LayerNorm/moments/variance" op: "Mean" input: "bert/encoder/layer_3/output/LayerNorm/moments/SquaredDifference" input: "bert/encoder/layer_3/output/LayerNorm/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_3/output/LayerNorm/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_3/output/LayerNorm/batchnorm/add" op: "Add" input: "bert/encoder/layer_3/output/LayerNorm/moments/variance" input: "bert/encoder/layer_3/output/LayerNorm/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_3/output/LayerNorm/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_3/output/LayerNorm/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_3/output/LayerNorm/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_3/output/LayerNorm/batchnorm/Rsqrt" input: "bert/encoder/layer_3/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_3/output/add" input: "bert/encoder/layer_3/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_3/output/LayerNorm/moments/mean" input: "bert/encoder/layer_3/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/output/LayerNorm/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_3/output/LayerNorm/beta/read" input: "bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/output/LayerNorm/batchnorm/add_1" op: "Add" input: "bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_1" input: "bert/encoder/layer_3/output/LayerNorm/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/Assign" op: "Assign" input: "bert/encoder/layer_4/attention/self/query/kernel" input: "bert/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/read" op: "Identity" input: "bert/encoder/layer_4/attention/self/query/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/self/query/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/self/query/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/attention/self/query/bias/Assign" op: "Assign" input: "bert/encoder/layer_4/attention/self/query/bias" input: "bert/encoder/layer_4/attention/self/query/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/attention/self/query/bias/read" op: "Identity" input: "bert/encoder/layer_4/attention/self/query/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/self/query/MatMul" op: "MatMul" input: "bert/encoder/layer_3/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_4/attention/self/query/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_4/attention/self/query/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_4/attention/self/query/MatMul" input: "bert/encoder/layer_4/attention/self/query/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/Assign" op: "Assign" input: "bert/encoder/layer_4/attention/self/key/kernel" input: "bert/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/read" op: "Identity" input: "bert/encoder/layer_4/attention/self/key/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/self/key/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/self/key/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/attention/self/key/bias/Assign" op: "Assign" input: "bert/encoder/layer_4/attention/self/key/bias" input: "bert/encoder/layer_4/attention/self/key/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/attention/self/key/bias/read" op: "Identity" input: "bert/encoder/layer_4/attention/self/key/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/self/key/MatMul" op: "MatMul" input: "bert/encoder/layer_3/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_4/attention/self/key/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_4/attention/self/key/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_4/attention/self/key/MatMul" input: "bert/encoder/layer_4/attention/self/key/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/Assign" op: "Assign" input: "bert/encoder/layer_4/attention/self/value/kernel" input: "bert/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/read" op: "Identity" input: "bert/encoder/layer_4/attention/self/value/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/self/value/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/self/value/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/attention/self/value/bias/Assign" op: "Assign" input: "bert/encoder/layer_4/attention/self/value/bias" input: "bert/encoder/layer_4/attention/self/value/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/attention/self/value/bias/read" op: "Identity" input: "bert/encoder/layer_4/attention/self/value/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/self/value/MatMul" op: "MatMul" input: "bert/encoder/layer_3/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_4/attention/self/value/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_4/attention/self/value/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_4/attention/self/value/MatMul" input: "bert/encoder/layer_4/attention/self/value/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_4/attention/self/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_4/attention/self/Reshape" op: "Reshape" input: "bert/encoder/layer_4/attention/self/query/BiasAdd" input: "bert/encoder/layer_4/attention/self/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_4/attention/self/transpose/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_4/attention/self/transpose" op: "Transpose" input: "bert/encoder/layer_4/attention/self/Reshape" input: "bert/encoder/layer_4/attention/self/transpose/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_4/attention/self/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_4/attention/self/Reshape_1" op: "Reshape" input: "bert/encoder/layer_4/attention/self/key/BiasAdd" input: "bert/encoder/layer_4/attention/self/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_4/attention/self/transpose_1/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_4/attention/self/transpose_1" op: "Transpose" input: "bert/encoder/layer_4/attention/self/Reshape_1" input: "bert/encoder/layer_4/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_4/attention/self/MatMul" op: "BatchMatMul" input: "bert/encoder/layer_4/attention/self/transpose" input: "bert/encoder/layer_4/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "bert/encoder/layer_4/attention/self/Mul/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.125 } } } } node { name: "bert/encoder/layer_4/attention/self/Mul" op: "Mul" input: "bert/encoder/layer_4/attention/self/MatMul" input: "bert/encoder/layer_4/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_4/attention/self/ExpandDims/dim" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_4/attention/self/ExpandDims" op: "ExpandDims" input: "bert/encoder/mul" input: "bert/encoder/layer_4/attention/self/ExpandDims/dim" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tdim" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_4/attention/self/sub/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_4/attention/self/sub" op: "Sub" input: "bert/encoder/layer_4/attention/self/sub/x" input: "bert/encoder/layer_4/attention/self/ExpandDims" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_4/attention/self/mul_1/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: -10000.0 } } } } node { name: "bert/encoder/layer_4/attention/self/mul_1" op: "Mul" input: "bert/encoder/layer_4/attention/self/sub" input: "bert/encoder/layer_4/attention/self/mul_1/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_4/attention/self/add" op: "Add" input: "bert/encoder/layer_4/attention/self/Mul" input: "bert/encoder/layer_4/attention/self/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_4/attention/self/Softmax" op: "Softmax" input: "bert/encoder/layer_4/attention/self/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_4/attention/self/dropout/keep_prob" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "bert/encoder/layer_4/attention/self/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "bert/encoder/layer_4/attention/self/dropout/random_uniform/min" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/self/dropout/random_uniform/max" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_4/attention/self/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_4/attention/self/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_4/attention/self/dropout/random_uniform/sub" op: "Sub" input: "bert/encoder/layer_4/attention/self/dropout/random_uniform/max" input: "bert/encoder/layer_4/attention/self/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/attention/self/dropout/random_uniform/mul" op: "Mul" input: "bert/encoder/layer_4/attention/self/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_4/attention/self/dropout/random_uniform/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_4/attention/self/dropout/random_uniform" op: "Add" input: "bert/encoder/layer_4/attention/self/dropout/random_uniform/mul" input: "bert/encoder/layer_4/attention/self/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_4/attention/self/dropout/add" op: "Add" input: "bert/encoder/layer_4/attention/self/dropout/keep_prob" input: "bert/encoder/layer_4/attention/self/dropout/random_uniform" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_4/attention/self/dropout/Floor" op: "Floor" input: "bert/encoder/layer_4/attention/self/dropout/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_4/attention/self/dropout/div" op: "RealDiv" input: "bert/encoder/layer_4/attention/self/Softmax" input: "bert/encoder/layer_4/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_4/attention/self/dropout/mul" op: "Mul" input: "bert/encoder/layer_4/attention/self/dropout/div" input: "bert/encoder/layer_4/attention/self/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_4/attention/self/Reshape_2/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_4/attention/self/Reshape_2" op: "Reshape" input: "bert/encoder/layer_4/attention/self/value/BiasAdd" input: "bert/encoder/layer_4/attention/self/Reshape_2/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_4/attention/self/transpose_2/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_4/attention/self/transpose_2" op: "Transpose" input: "bert/encoder/layer_4/attention/self/Reshape_2" input: "bert/encoder/layer_4/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_4/attention/self/MatMul_1" op: "BatchMatMul" input: "bert/encoder/layer_4/attention/self/dropout/mul" input: "bert/encoder/layer_4/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "bert/encoder/layer_4/attention/self/transpose_3/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_4/attention/self/transpose_3" op: "Transpose" input: "bert/encoder/layer_4/attention/self/MatMul_1" input: "bert/encoder/layer_4/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_4/attention/self/Reshape_3/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_4/attention/self/Reshape_3" op: "Reshape" input: "bert/encoder/layer_4/attention/self/transpose_3" input: "bert/encoder/layer_4/attention/self/Reshape_3/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/Assign" op: "Assign" input: "bert/encoder/layer_4/attention/output/dense/kernel" input: "bert/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/read" op: "Identity" input: "bert/encoder/layer_4/attention/output/dense/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/output/dense/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/attention/output/dense/bias/Assign" op: "Assign" input: "bert/encoder/layer_4/attention/output/dense/bias" input: "bert/encoder/layer_4/attention/output/dense/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/attention/output/dense/bias/read" op: "Identity" input: "bert/encoder/layer_4/attention/output/dense/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_4/attention/self/Reshape_3" input: "bert/encoder/layer_4/attention/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_4/attention/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_4/attention/output/dense/MatMul" input: "bert/encoder/layer_4/attention/output/dense/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_4/attention/output/dropout/keep_prob" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "bert/encoder/layer_4/attention/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_4/attention/output/dropout/random_uniform/min" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/output/dropout/random_uniform/max" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_4/attention/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_4/attention/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_4/attention/output/dropout/random_uniform/sub" op: "Sub" input: "bert/encoder/layer_4/attention/output/dropout/random_uniform/max" input: "bert/encoder/layer_4/attention/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/attention/output/dropout/random_uniform/mul" op: "Mul" input: "bert/encoder/layer_4/attention/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_4/attention/output/dropout/random_uniform/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/output/dropout/random_uniform" op: "Add" input: "bert/encoder/layer_4/attention/output/dropout/random_uniform/mul" input: "bert/encoder/layer_4/attention/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/output/dropout/add" op: "Add" input: "bert/encoder/layer_4/attention/output/dropout/keep_prob" input: "bert/encoder/layer_4/attention/output/dropout/random_uniform" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/output/dropout/Floor" op: "Floor" input: "bert/encoder/layer_4/attention/output/dropout/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/output/dropout/div" op: "RealDiv" input: "bert/encoder/layer_4/attention/output/dense/BiasAdd" input: "bert/encoder/layer_4/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/output/dropout/mul" op: "Mul" input: "bert/encoder/layer_4/attention/output/dropout/div" input: "bert/encoder/layer_4/attention/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/output/add" op: "Add" input: "bert/encoder/layer_4/attention/output/dropout/mul" input: "bert/encoder/layer_3/output/LayerNorm/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/output/LayerNorm/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/output/LayerNorm/beta" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/attention/output/LayerNorm/beta/Assign" op: "Assign" input: "bert/encoder/layer_4/attention/output/LayerNorm/beta" input: "bert/encoder/layer_4/attention/output/LayerNorm/beta/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/attention/output/LayerNorm/beta/read" op: "Identity" input: "bert/encoder/layer_4/attention/output/LayerNorm/beta" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_4/attention/output/LayerNorm/gamma" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/Assign" op: "Assign" input: "bert/encoder/layer_4/attention/output/LayerNorm/gamma" input: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/Initializer/ones" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/read" op: "Identity" input: "bert/encoder/layer_4/attention/output/LayerNorm/gamma" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/output/LayerNorm/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_4/attention/output/LayerNorm/moments/mean" op: "Mean" input: "bert/encoder/layer_4/attention/output/add" input: "bert/encoder/layer_4/attention/output/LayerNorm/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_4/attention/output/LayerNorm/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_4/attention/output/LayerNorm/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_4/attention/output/add" input: "bert/encoder/layer_4/attention/output/LayerNorm/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/output/LayerNorm/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_4/attention/output/LayerNorm/moments/variance" op: "Mean" input: "bert/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference" input: "bert/encoder/layer_4/attention/output/LayerNorm/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/add" op: "Add" input: "bert/encoder/layer_4/attention/output/LayerNorm/moments/variance" input: "bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/Rsqrt" input: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_4/attention/output/add" input: "bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_4/attention/output/LayerNorm/moments/mean" input: "bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_4/attention/output/LayerNorm/beta/read" input: "bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_1" op: "Add" input: "bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_1" input: "bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/Assign" op: "Assign" input: "bert/encoder/layer_4/intermediate/dense/kernel" input: "bert/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/read" op: "Identity" input: "bert/encoder/layer_4/intermediate/dense/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_4/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_4/intermediate/dense/bias/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/intermediate/dense/bias/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_4/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_4/intermediate/dense/bias/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_4/intermediate/dense/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/intermediate/dense/bias/Assign" op: "Assign" input: "bert/encoder/layer_4/intermediate/dense/bias" input: "bert/encoder/layer_4/intermediate/dense/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/intermediate/dense/bias/read" op: "Identity" input: "bert/encoder/layer_4/intermediate/dense/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_4/intermediate/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_4/intermediate/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_4/intermediate/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_4/intermediate/dense/MatMul" input: "bert/encoder/layer_4/intermediate/dense/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_4/intermediate/dense/Sqrt/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "bert/encoder/layer_4/intermediate/dense/Sqrt" op: "Sqrt" input: "bert/encoder/layer_4/intermediate/dense/Sqrt/x" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/intermediate/dense/truediv" op: "RealDiv" input: "bert/encoder/layer_4/intermediate/dense/BiasAdd" input: "bert/encoder/layer_4/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_4/intermediate/dense/Erf" op: "Erf" input: "bert/encoder/layer_4/intermediate/dense/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_4/intermediate/dense/add/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_4/intermediate/dense/add" op: "Add" input: "bert/encoder/layer_4/intermediate/dense/add/x" input: "bert/encoder/layer_4/intermediate/dense/Erf" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_4/intermediate/dense/mul/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.5 } } } } node { name: "bert/encoder/layer_4/intermediate/dense/mul" op: "Mul" input: "bert/encoder/layer_4/intermediate/dense/mul/x" input: "bert/encoder/layer_4/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_4/intermediate/dense/mul_1" op: "Mul" input: "bert/encoder/layer_4/intermediate/dense/BiasAdd" input: "bert/encoder/layer_4/intermediate/dense/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/output/dense/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/output/dense/kernel/Assign" op: "Assign" input: "bert/encoder/layer_4/output/dense/kernel" input: "bert/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/output/dense/kernel/read" op: "Identity" input: "bert/encoder/layer_4/output/dense/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/output/dense/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/output/dense/bias/Assign" op: "Assign" input: "bert/encoder/layer_4/output/dense/bias" input: "bert/encoder/layer_4/output/dense/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/output/dense/bias/read" op: "Identity" input: "bert/encoder/layer_4/output/dense/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_4/intermediate/dense/mul_1" input: "bert/encoder/layer_4/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_4/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_4/output/dense/MatMul" input: "bert/encoder/layer_4/output/dense/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_4/output/dropout/keep_prob" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "bert/encoder/layer_4/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_4/output/dropout/random_uniform/min" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/output/dropout/random_uniform/max" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_4/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_4/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_4/output/dropout/random_uniform/sub" op: "Sub" input: "bert/encoder/layer_4/output/dropout/random_uniform/max" input: "bert/encoder/layer_4/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/output/dropout/random_uniform/mul" op: "Mul" input: "bert/encoder/layer_4/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_4/output/dropout/random_uniform/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/output/dropout/random_uniform" op: "Add" input: "bert/encoder/layer_4/output/dropout/random_uniform/mul" input: "bert/encoder/layer_4/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/output/dropout/add" op: "Add" input: "bert/encoder/layer_4/output/dropout/keep_prob" input: "bert/encoder/layer_4/output/dropout/random_uniform" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/output/dropout/Floor" op: "Floor" input: "bert/encoder/layer_4/output/dropout/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/output/dropout/div" op: "RealDiv" input: "bert/encoder/layer_4/output/dense/BiasAdd" input: "bert/encoder/layer_4/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/output/dropout/mul" op: "Mul" input: "bert/encoder/layer_4/output/dropout/div" input: "bert/encoder/layer_4/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/output/add" op: "Add" input: "bert/encoder/layer_4/output/dropout/mul" input: "bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/output/LayerNorm/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/output/LayerNorm/beta" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/output/LayerNorm/beta/Assign" op: "Assign" input: "bert/encoder/layer_4/output/LayerNorm/beta" input: "bert/encoder/layer_4/output/LayerNorm/beta/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/output/LayerNorm/beta/read" op: "Identity" input: "bert/encoder/layer_4/output/LayerNorm/beta" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/output/LayerNorm/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_4/output/LayerNorm/gamma" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/output/LayerNorm/gamma/Assign" op: "Assign" input: "bert/encoder/layer_4/output/LayerNorm/gamma" input: "bert/encoder/layer_4/output/LayerNorm/gamma/Initializer/ones" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/output/LayerNorm/gamma/read" op: "Identity" input: "bert/encoder/layer_4/output/LayerNorm/gamma" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/output/LayerNorm/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_4/output/LayerNorm/moments/mean" op: "Mean" input: "bert/encoder/layer_4/output/add" input: "bert/encoder/layer_4/output/LayerNorm/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_4/output/LayerNorm/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_4/output/LayerNorm/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_4/output/LayerNorm/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_4/output/add" input: "bert/encoder/layer_4/output/LayerNorm/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/output/LayerNorm/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_4/output/LayerNorm/moments/variance" op: "Mean" input: "bert/encoder/layer_4/output/LayerNorm/moments/SquaredDifference" input: "bert/encoder/layer_4/output/LayerNorm/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_4/output/LayerNorm/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_4/output/LayerNorm/batchnorm/add" op: "Add" input: "bert/encoder/layer_4/output/LayerNorm/moments/variance" input: "bert/encoder/layer_4/output/LayerNorm/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_4/output/LayerNorm/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_4/output/LayerNorm/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_4/output/LayerNorm/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_4/output/LayerNorm/batchnorm/Rsqrt" input: "bert/encoder/layer_4/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_4/output/add" input: "bert/encoder/layer_4/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_4/output/LayerNorm/moments/mean" input: "bert/encoder/layer_4/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/output/LayerNorm/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_4/output/LayerNorm/beta/read" input: "bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/output/LayerNorm/batchnorm/add_1" op: "Add" input: "bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_1" input: "bert/encoder/layer_4/output/LayerNorm/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/Assign" op: "Assign" input: "bert/encoder/layer_5/attention/self/query/kernel" input: "bert/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/read" op: "Identity" input: "bert/encoder/layer_5/attention/self/query/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/self/query/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/self/query/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/attention/self/query/bias/Assign" op: "Assign" input: "bert/encoder/layer_5/attention/self/query/bias" input: "bert/encoder/layer_5/attention/self/query/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/attention/self/query/bias/read" op: "Identity" input: "bert/encoder/layer_5/attention/self/query/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/self/query/MatMul" op: "MatMul" input: "bert/encoder/layer_4/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_5/attention/self/query/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_5/attention/self/query/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_5/attention/self/query/MatMul" input: "bert/encoder/layer_5/attention/self/query/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/Assign" op: "Assign" input: "bert/encoder/layer_5/attention/self/key/kernel" input: "bert/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/read" op: "Identity" input: "bert/encoder/layer_5/attention/self/key/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/self/key/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/self/key/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/attention/self/key/bias/Assign" op: "Assign" input: "bert/encoder/layer_5/attention/self/key/bias" input: "bert/encoder/layer_5/attention/self/key/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/attention/self/key/bias/read" op: "Identity" input: "bert/encoder/layer_5/attention/self/key/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/self/key/MatMul" op: "MatMul" input: "bert/encoder/layer_4/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_5/attention/self/key/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_5/attention/self/key/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_5/attention/self/key/MatMul" input: "bert/encoder/layer_5/attention/self/key/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/Assign" op: "Assign" input: "bert/encoder/layer_5/attention/self/value/kernel" input: "bert/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/read" op: "Identity" input: "bert/encoder/layer_5/attention/self/value/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/self/value/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/self/value/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/attention/self/value/bias/Assign" op: "Assign" input: "bert/encoder/layer_5/attention/self/value/bias" input: "bert/encoder/layer_5/attention/self/value/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/attention/self/value/bias/read" op: "Identity" input: "bert/encoder/layer_5/attention/self/value/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/self/value/MatMul" op: "MatMul" input: "bert/encoder/layer_4/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_5/attention/self/value/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_5/attention/self/value/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_5/attention/self/value/MatMul" input: "bert/encoder/layer_5/attention/self/value/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_5/attention/self/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_5/attention/self/Reshape" op: "Reshape" input: "bert/encoder/layer_5/attention/self/query/BiasAdd" input: "bert/encoder/layer_5/attention/self/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_5/attention/self/transpose/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_5/attention/self/transpose" op: "Transpose" input: "bert/encoder/layer_5/attention/self/Reshape" input: "bert/encoder/layer_5/attention/self/transpose/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_5/attention/self/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_5/attention/self/Reshape_1" op: "Reshape" input: "bert/encoder/layer_5/attention/self/key/BiasAdd" input: "bert/encoder/layer_5/attention/self/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_5/attention/self/transpose_1/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_5/attention/self/transpose_1" op: "Transpose" input: "bert/encoder/layer_5/attention/self/Reshape_1" input: "bert/encoder/layer_5/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_5/attention/self/MatMul" op: "BatchMatMul" input: "bert/encoder/layer_5/attention/self/transpose" input: "bert/encoder/layer_5/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "bert/encoder/layer_5/attention/self/Mul/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.125 } } } } node { name: "bert/encoder/layer_5/attention/self/Mul" op: "Mul" input: "bert/encoder/layer_5/attention/self/MatMul" input: "bert/encoder/layer_5/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_5/attention/self/ExpandDims/dim" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_5/attention/self/ExpandDims" op: "ExpandDims" input: "bert/encoder/mul" input: "bert/encoder/layer_5/attention/self/ExpandDims/dim" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tdim" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_5/attention/self/sub/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_5/attention/self/sub" op: "Sub" input: "bert/encoder/layer_5/attention/self/sub/x" input: "bert/encoder/layer_5/attention/self/ExpandDims" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_5/attention/self/mul_1/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: -10000.0 } } } } node { name: "bert/encoder/layer_5/attention/self/mul_1" op: "Mul" input: "bert/encoder/layer_5/attention/self/sub" input: "bert/encoder/layer_5/attention/self/mul_1/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_5/attention/self/add" op: "Add" input: "bert/encoder/layer_5/attention/self/Mul" input: "bert/encoder/layer_5/attention/self/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_5/attention/self/Softmax" op: "Softmax" input: "bert/encoder/layer_5/attention/self/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_5/attention/self/dropout/keep_prob" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "bert/encoder/layer_5/attention/self/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "bert/encoder/layer_5/attention/self/dropout/random_uniform/min" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/self/dropout/random_uniform/max" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_5/attention/self/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_5/attention/self/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_5/attention/self/dropout/random_uniform/sub" op: "Sub" input: "bert/encoder/layer_5/attention/self/dropout/random_uniform/max" input: "bert/encoder/layer_5/attention/self/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/attention/self/dropout/random_uniform/mul" op: "Mul" input: "bert/encoder/layer_5/attention/self/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_5/attention/self/dropout/random_uniform/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_5/attention/self/dropout/random_uniform" op: "Add" input: "bert/encoder/layer_5/attention/self/dropout/random_uniform/mul" input: "bert/encoder/layer_5/attention/self/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_5/attention/self/dropout/add" op: "Add" input: "bert/encoder/layer_5/attention/self/dropout/keep_prob" input: "bert/encoder/layer_5/attention/self/dropout/random_uniform" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_5/attention/self/dropout/Floor" op: "Floor" input: "bert/encoder/layer_5/attention/self/dropout/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_5/attention/self/dropout/div" op: "RealDiv" input: "bert/encoder/layer_5/attention/self/Softmax" input: "bert/encoder/layer_5/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_5/attention/self/dropout/mul" op: "Mul" input: "bert/encoder/layer_5/attention/self/dropout/div" input: "bert/encoder/layer_5/attention/self/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_5/attention/self/Reshape_2/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_5/attention/self/Reshape_2" op: "Reshape" input: "bert/encoder/layer_5/attention/self/value/BiasAdd" input: "bert/encoder/layer_5/attention/self/Reshape_2/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_5/attention/self/transpose_2/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_5/attention/self/transpose_2" op: "Transpose" input: "bert/encoder/layer_5/attention/self/Reshape_2" input: "bert/encoder/layer_5/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_5/attention/self/MatMul_1" op: "BatchMatMul" input: "bert/encoder/layer_5/attention/self/dropout/mul" input: "bert/encoder/layer_5/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "bert/encoder/layer_5/attention/self/transpose_3/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_5/attention/self/transpose_3" op: "Transpose" input: "bert/encoder/layer_5/attention/self/MatMul_1" input: "bert/encoder/layer_5/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_5/attention/self/Reshape_3/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_5/attention/self/Reshape_3" op: "Reshape" input: "bert/encoder/layer_5/attention/self/transpose_3" input: "bert/encoder/layer_5/attention/self/Reshape_3/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/Assign" op: "Assign" input: "bert/encoder/layer_5/attention/output/dense/kernel" input: "bert/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/read" op: "Identity" input: "bert/encoder/layer_5/attention/output/dense/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/output/dense/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/attention/output/dense/bias/Assign" op: "Assign" input: "bert/encoder/layer_5/attention/output/dense/bias" input: "bert/encoder/layer_5/attention/output/dense/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/attention/output/dense/bias/read" op: "Identity" input: "bert/encoder/layer_5/attention/output/dense/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_5/attention/self/Reshape_3" input: "bert/encoder/layer_5/attention/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_5/attention/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_5/attention/output/dense/MatMul" input: "bert/encoder/layer_5/attention/output/dense/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_5/attention/output/dropout/keep_prob" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "bert/encoder/layer_5/attention/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_5/attention/output/dropout/random_uniform/min" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/output/dropout/random_uniform/max" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_5/attention/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_5/attention/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_5/attention/output/dropout/random_uniform/sub" op: "Sub" input: "bert/encoder/layer_5/attention/output/dropout/random_uniform/max" input: "bert/encoder/layer_5/attention/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/attention/output/dropout/random_uniform/mul" op: "Mul" input: "bert/encoder/layer_5/attention/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_5/attention/output/dropout/random_uniform/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/output/dropout/random_uniform" op: "Add" input: "bert/encoder/layer_5/attention/output/dropout/random_uniform/mul" input: "bert/encoder/layer_5/attention/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/output/dropout/add" op: "Add" input: "bert/encoder/layer_5/attention/output/dropout/keep_prob" input: "bert/encoder/layer_5/attention/output/dropout/random_uniform" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/output/dropout/Floor" op: "Floor" input: "bert/encoder/layer_5/attention/output/dropout/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/output/dropout/div" op: "RealDiv" input: "bert/encoder/layer_5/attention/output/dense/BiasAdd" input: "bert/encoder/layer_5/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/output/dropout/mul" op: "Mul" input: "bert/encoder/layer_5/attention/output/dropout/div" input: "bert/encoder/layer_5/attention/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/output/add" op: "Add" input: "bert/encoder/layer_5/attention/output/dropout/mul" input: "bert/encoder/layer_4/output/LayerNorm/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/output/LayerNorm/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/output/LayerNorm/beta" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/attention/output/LayerNorm/beta/Assign" op: "Assign" input: "bert/encoder/layer_5/attention/output/LayerNorm/beta" input: "bert/encoder/layer_5/attention/output/LayerNorm/beta/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/attention/output/LayerNorm/beta/read" op: "Identity" input: "bert/encoder/layer_5/attention/output/LayerNorm/beta" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_5/attention/output/LayerNorm/gamma" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/Assign" op: "Assign" input: "bert/encoder/layer_5/attention/output/LayerNorm/gamma" input: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/Initializer/ones" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/read" op: "Identity" input: "bert/encoder/layer_5/attention/output/LayerNorm/gamma" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/output/LayerNorm/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_5/attention/output/LayerNorm/moments/mean" op: "Mean" input: "bert/encoder/layer_5/attention/output/add" input: "bert/encoder/layer_5/attention/output/LayerNorm/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_5/attention/output/LayerNorm/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_5/attention/output/LayerNorm/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_5/attention/output/add" input: "bert/encoder/layer_5/attention/output/LayerNorm/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/output/LayerNorm/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_5/attention/output/LayerNorm/moments/variance" op: "Mean" input: "bert/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference" input: "bert/encoder/layer_5/attention/output/LayerNorm/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/add" op: "Add" input: "bert/encoder/layer_5/attention/output/LayerNorm/moments/variance" input: "bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/Rsqrt" input: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_5/attention/output/add" input: "bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_5/attention/output/LayerNorm/moments/mean" input: "bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_5/attention/output/LayerNorm/beta/read" input: "bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_1" op: "Add" input: "bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_1" input: "bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/Assign" op: "Assign" input: "bert/encoder/layer_5/intermediate/dense/kernel" input: "bert/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/read" op: "Identity" input: "bert/encoder/layer_5/intermediate/dense/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_5/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_5/intermediate/dense/bias/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/intermediate/dense/bias/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_5/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_5/intermediate/dense/bias/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_5/intermediate/dense/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/intermediate/dense/bias/Assign" op: "Assign" input: "bert/encoder/layer_5/intermediate/dense/bias" input: "bert/encoder/layer_5/intermediate/dense/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/intermediate/dense/bias/read" op: "Identity" input: "bert/encoder/layer_5/intermediate/dense/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_5/intermediate/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_5/intermediate/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_5/intermediate/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_5/intermediate/dense/MatMul" input: "bert/encoder/layer_5/intermediate/dense/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_5/intermediate/dense/Sqrt/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "bert/encoder/layer_5/intermediate/dense/Sqrt" op: "Sqrt" input: "bert/encoder/layer_5/intermediate/dense/Sqrt/x" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/intermediate/dense/truediv" op: "RealDiv" input: "bert/encoder/layer_5/intermediate/dense/BiasAdd" input: "bert/encoder/layer_5/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_5/intermediate/dense/Erf" op: "Erf" input: "bert/encoder/layer_5/intermediate/dense/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_5/intermediate/dense/add/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_5/intermediate/dense/add" op: "Add" input: "bert/encoder/layer_5/intermediate/dense/add/x" input: "bert/encoder/layer_5/intermediate/dense/Erf" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_5/intermediate/dense/mul/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.5 } } } } node { name: "bert/encoder/layer_5/intermediate/dense/mul" op: "Mul" input: "bert/encoder/layer_5/intermediate/dense/mul/x" input: "bert/encoder/layer_5/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_5/intermediate/dense/mul_1" op: "Mul" input: "bert/encoder/layer_5/intermediate/dense/BiasAdd" input: "bert/encoder/layer_5/intermediate/dense/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/output/dense/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/output/dense/kernel/Assign" op: "Assign" input: "bert/encoder/layer_5/output/dense/kernel" input: "bert/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/output/dense/kernel/read" op: "Identity" input: "bert/encoder/layer_5/output/dense/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/output/dense/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/output/dense/bias/Assign" op: "Assign" input: "bert/encoder/layer_5/output/dense/bias" input: "bert/encoder/layer_5/output/dense/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/output/dense/bias/read" op: "Identity" input: "bert/encoder/layer_5/output/dense/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_5/intermediate/dense/mul_1" input: "bert/encoder/layer_5/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_5/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_5/output/dense/MatMul" input: "bert/encoder/layer_5/output/dense/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_5/output/dropout/keep_prob" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "bert/encoder/layer_5/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_5/output/dropout/random_uniform/min" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/output/dropout/random_uniform/max" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_5/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_5/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_5/output/dropout/random_uniform/sub" op: "Sub" input: "bert/encoder/layer_5/output/dropout/random_uniform/max" input: "bert/encoder/layer_5/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/output/dropout/random_uniform/mul" op: "Mul" input: "bert/encoder/layer_5/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_5/output/dropout/random_uniform/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/output/dropout/random_uniform" op: "Add" input: "bert/encoder/layer_5/output/dropout/random_uniform/mul" input: "bert/encoder/layer_5/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/output/dropout/add" op: "Add" input: "bert/encoder/layer_5/output/dropout/keep_prob" input: "bert/encoder/layer_5/output/dropout/random_uniform" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/output/dropout/Floor" op: "Floor" input: "bert/encoder/layer_5/output/dropout/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/output/dropout/div" op: "RealDiv" input: "bert/encoder/layer_5/output/dense/BiasAdd" input: "bert/encoder/layer_5/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/output/dropout/mul" op: "Mul" input: "bert/encoder/layer_5/output/dropout/div" input: "bert/encoder/layer_5/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/output/add" op: "Add" input: "bert/encoder/layer_5/output/dropout/mul" input: "bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/output/LayerNorm/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/output/LayerNorm/beta" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/output/LayerNorm/beta/Assign" op: "Assign" input: "bert/encoder/layer_5/output/LayerNorm/beta" input: "bert/encoder/layer_5/output/LayerNorm/beta/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/output/LayerNorm/beta/read" op: "Identity" input: "bert/encoder/layer_5/output/LayerNorm/beta" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/output/LayerNorm/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_5/output/LayerNorm/gamma" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/output/LayerNorm/gamma/Assign" op: "Assign" input: "bert/encoder/layer_5/output/LayerNorm/gamma" input: "bert/encoder/layer_5/output/LayerNorm/gamma/Initializer/ones" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/output/LayerNorm/gamma/read" op: "Identity" input: "bert/encoder/layer_5/output/LayerNorm/gamma" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/output/LayerNorm/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_5/output/LayerNorm/moments/mean" op: "Mean" input: "bert/encoder/layer_5/output/add" input: "bert/encoder/layer_5/output/LayerNorm/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_5/output/LayerNorm/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_5/output/LayerNorm/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_5/output/LayerNorm/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_5/output/add" input: "bert/encoder/layer_5/output/LayerNorm/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/output/LayerNorm/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_5/output/LayerNorm/moments/variance" op: "Mean" input: "bert/encoder/layer_5/output/LayerNorm/moments/SquaredDifference" input: "bert/encoder/layer_5/output/LayerNorm/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_5/output/LayerNorm/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_5/output/LayerNorm/batchnorm/add" op: "Add" input: "bert/encoder/layer_5/output/LayerNorm/moments/variance" input: "bert/encoder/layer_5/output/LayerNorm/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_5/output/LayerNorm/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_5/output/LayerNorm/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_5/output/LayerNorm/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_5/output/LayerNorm/batchnorm/Rsqrt" input: "bert/encoder/layer_5/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_5/output/add" input: "bert/encoder/layer_5/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_5/output/LayerNorm/moments/mean" input: "bert/encoder/layer_5/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/output/LayerNorm/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_5/output/LayerNorm/beta/read" input: "bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/output/LayerNorm/batchnorm/add_1" op: "Add" input: "bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_1" input: "bert/encoder/layer_5/output/LayerNorm/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/Assign" op: "Assign" input: "bert/encoder/layer_6/attention/self/query/kernel" input: "bert/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/read" op: "Identity" input: "bert/encoder/layer_6/attention/self/query/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/self/query/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/self/query/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/attention/self/query/bias/Assign" op: "Assign" input: "bert/encoder/layer_6/attention/self/query/bias" input: "bert/encoder/layer_6/attention/self/query/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/attention/self/query/bias/read" op: "Identity" input: "bert/encoder/layer_6/attention/self/query/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/self/query/MatMul" op: "MatMul" input: "bert/encoder/layer_5/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_6/attention/self/query/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_6/attention/self/query/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_6/attention/self/query/MatMul" input: "bert/encoder/layer_6/attention/self/query/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/Assign" op: "Assign" input: "bert/encoder/layer_6/attention/self/key/kernel" input: "bert/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/read" op: "Identity" input: "bert/encoder/layer_6/attention/self/key/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/self/key/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/self/key/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/attention/self/key/bias/Assign" op: "Assign" input: "bert/encoder/layer_6/attention/self/key/bias" input: "bert/encoder/layer_6/attention/self/key/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/attention/self/key/bias/read" op: "Identity" input: "bert/encoder/layer_6/attention/self/key/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/self/key/MatMul" op: "MatMul" input: "bert/encoder/layer_5/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_6/attention/self/key/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_6/attention/self/key/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_6/attention/self/key/MatMul" input: "bert/encoder/layer_6/attention/self/key/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/Assign" op: "Assign" input: "bert/encoder/layer_6/attention/self/value/kernel" input: "bert/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/read" op: "Identity" input: "bert/encoder/layer_6/attention/self/value/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/self/value/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/self/value/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/attention/self/value/bias/Assign" op: "Assign" input: "bert/encoder/layer_6/attention/self/value/bias" input: "bert/encoder/layer_6/attention/self/value/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/attention/self/value/bias/read" op: "Identity" input: "bert/encoder/layer_6/attention/self/value/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/self/value/MatMul" op: "MatMul" input: "bert/encoder/layer_5/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_6/attention/self/value/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_6/attention/self/value/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_6/attention/self/value/MatMul" input: "bert/encoder/layer_6/attention/self/value/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_6/attention/self/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_6/attention/self/Reshape" op: "Reshape" input: "bert/encoder/layer_6/attention/self/query/BiasAdd" input: "bert/encoder/layer_6/attention/self/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_6/attention/self/transpose/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_6/attention/self/transpose" op: "Transpose" input: "bert/encoder/layer_6/attention/self/Reshape" input: "bert/encoder/layer_6/attention/self/transpose/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_6/attention/self/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_6/attention/self/Reshape_1" op: "Reshape" input: "bert/encoder/layer_6/attention/self/key/BiasAdd" input: "bert/encoder/layer_6/attention/self/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_6/attention/self/transpose_1/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_6/attention/self/transpose_1" op: "Transpose" input: "bert/encoder/layer_6/attention/self/Reshape_1" input: "bert/encoder/layer_6/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_6/attention/self/MatMul" op: "BatchMatMul" input: "bert/encoder/layer_6/attention/self/transpose" input: "bert/encoder/layer_6/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "bert/encoder/layer_6/attention/self/Mul/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.125 } } } } node { name: "bert/encoder/layer_6/attention/self/Mul" op: "Mul" input: "bert/encoder/layer_6/attention/self/MatMul" input: "bert/encoder/layer_6/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_6/attention/self/ExpandDims/dim" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_6/attention/self/ExpandDims" op: "ExpandDims" input: "bert/encoder/mul" input: "bert/encoder/layer_6/attention/self/ExpandDims/dim" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tdim" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_6/attention/self/sub/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_6/attention/self/sub" op: "Sub" input: "bert/encoder/layer_6/attention/self/sub/x" input: "bert/encoder/layer_6/attention/self/ExpandDims" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_6/attention/self/mul_1/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: -10000.0 } } } } node { name: "bert/encoder/layer_6/attention/self/mul_1" op: "Mul" input: "bert/encoder/layer_6/attention/self/sub" input: "bert/encoder/layer_6/attention/self/mul_1/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_6/attention/self/add" op: "Add" input: "bert/encoder/layer_6/attention/self/Mul" input: "bert/encoder/layer_6/attention/self/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_6/attention/self/Softmax" op: "Softmax" input: "bert/encoder/layer_6/attention/self/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_6/attention/self/dropout/keep_prob" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "bert/encoder/layer_6/attention/self/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "bert/encoder/layer_6/attention/self/dropout/random_uniform/min" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/self/dropout/random_uniform/max" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_6/attention/self/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_6/attention/self/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_6/attention/self/dropout/random_uniform/sub" op: "Sub" input: "bert/encoder/layer_6/attention/self/dropout/random_uniform/max" input: "bert/encoder/layer_6/attention/self/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/attention/self/dropout/random_uniform/mul" op: "Mul" input: "bert/encoder/layer_6/attention/self/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_6/attention/self/dropout/random_uniform/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_6/attention/self/dropout/random_uniform" op: "Add" input: "bert/encoder/layer_6/attention/self/dropout/random_uniform/mul" input: "bert/encoder/layer_6/attention/self/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_6/attention/self/dropout/add" op: "Add" input: "bert/encoder/layer_6/attention/self/dropout/keep_prob" input: "bert/encoder/layer_6/attention/self/dropout/random_uniform" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_6/attention/self/dropout/Floor" op: "Floor" input: "bert/encoder/layer_6/attention/self/dropout/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_6/attention/self/dropout/div" op: "RealDiv" input: "bert/encoder/layer_6/attention/self/Softmax" input: "bert/encoder/layer_6/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_6/attention/self/dropout/mul" op: "Mul" input: "bert/encoder/layer_6/attention/self/dropout/div" input: "bert/encoder/layer_6/attention/self/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_6/attention/self/Reshape_2/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_6/attention/self/Reshape_2" op: "Reshape" input: "bert/encoder/layer_6/attention/self/value/BiasAdd" input: "bert/encoder/layer_6/attention/self/Reshape_2/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_6/attention/self/transpose_2/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_6/attention/self/transpose_2" op: "Transpose" input: "bert/encoder/layer_6/attention/self/Reshape_2" input: "bert/encoder/layer_6/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_6/attention/self/MatMul_1" op: "BatchMatMul" input: "bert/encoder/layer_6/attention/self/dropout/mul" input: "bert/encoder/layer_6/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "bert/encoder/layer_6/attention/self/transpose_3/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_6/attention/self/transpose_3" op: "Transpose" input: "bert/encoder/layer_6/attention/self/MatMul_1" input: "bert/encoder/layer_6/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_6/attention/self/Reshape_3/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_6/attention/self/Reshape_3" op: "Reshape" input: "bert/encoder/layer_6/attention/self/transpose_3" input: "bert/encoder/layer_6/attention/self/Reshape_3/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/Assign" op: "Assign" input: "bert/encoder/layer_6/attention/output/dense/kernel" input: "bert/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/read" op: "Identity" input: "bert/encoder/layer_6/attention/output/dense/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/output/dense/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/attention/output/dense/bias/Assign" op: "Assign" input: "bert/encoder/layer_6/attention/output/dense/bias" input: "bert/encoder/layer_6/attention/output/dense/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/attention/output/dense/bias/read" op: "Identity" input: "bert/encoder/layer_6/attention/output/dense/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_6/attention/self/Reshape_3" input: "bert/encoder/layer_6/attention/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_6/attention/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_6/attention/output/dense/MatMul" input: "bert/encoder/layer_6/attention/output/dense/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_6/attention/output/dropout/keep_prob" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "bert/encoder/layer_6/attention/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_6/attention/output/dropout/random_uniform/min" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/output/dropout/random_uniform/max" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_6/attention/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_6/attention/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_6/attention/output/dropout/random_uniform/sub" op: "Sub" input: "bert/encoder/layer_6/attention/output/dropout/random_uniform/max" input: "bert/encoder/layer_6/attention/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/attention/output/dropout/random_uniform/mul" op: "Mul" input: "bert/encoder/layer_6/attention/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_6/attention/output/dropout/random_uniform/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/output/dropout/random_uniform" op: "Add" input: "bert/encoder/layer_6/attention/output/dropout/random_uniform/mul" input: "bert/encoder/layer_6/attention/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/output/dropout/add" op: "Add" input: "bert/encoder/layer_6/attention/output/dropout/keep_prob" input: "bert/encoder/layer_6/attention/output/dropout/random_uniform" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/output/dropout/Floor" op: "Floor" input: "bert/encoder/layer_6/attention/output/dropout/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/output/dropout/div" op: "RealDiv" input: "bert/encoder/layer_6/attention/output/dense/BiasAdd" input: "bert/encoder/layer_6/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/output/dropout/mul" op: "Mul" input: "bert/encoder/layer_6/attention/output/dropout/div" input: "bert/encoder/layer_6/attention/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/output/add" op: "Add" input: "bert/encoder/layer_6/attention/output/dropout/mul" input: "bert/encoder/layer_5/output/LayerNorm/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/output/LayerNorm/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/output/LayerNorm/beta" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/attention/output/LayerNorm/beta/Assign" op: "Assign" input: "bert/encoder/layer_6/attention/output/LayerNorm/beta" input: "bert/encoder/layer_6/attention/output/LayerNorm/beta/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/attention/output/LayerNorm/beta/read" op: "Identity" input: "bert/encoder/layer_6/attention/output/LayerNorm/beta" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_6/attention/output/LayerNorm/gamma" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/Assign" op: "Assign" input: "bert/encoder/layer_6/attention/output/LayerNorm/gamma" input: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/Initializer/ones" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/read" op: "Identity" input: "bert/encoder/layer_6/attention/output/LayerNorm/gamma" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/output/LayerNorm/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_6/attention/output/LayerNorm/moments/mean" op: "Mean" input: "bert/encoder/layer_6/attention/output/add" input: "bert/encoder/layer_6/attention/output/LayerNorm/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_6/attention/output/LayerNorm/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_6/attention/output/LayerNorm/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_6/attention/output/add" input: "bert/encoder/layer_6/attention/output/LayerNorm/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/output/LayerNorm/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_6/attention/output/LayerNorm/moments/variance" op: "Mean" input: "bert/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference" input: "bert/encoder/layer_6/attention/output/LayerNorm/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/add" op: "Add" input: "bert/encoder/layer_6/attention/output/LayerNorm/moments/variance" input: "bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/Rsqrt" input: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_6/attention/output/add" input: "bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_6/attention/output/LayerNorm/moments/mean" input: "bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_6/attention/output/LayerNorm/beta/read" input: "bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_1" op: "Add" input: "bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_1" input: "bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/Assign" op: "Assign" input: "bert/encoder/layer_6/intermediate/dense/kernel" input: "bert/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/read" op: "Identity" input: "bert/encoder/layer_6/intermediate/dense/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_6/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_6/intermediate/dense/bias/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/intermediate/dense/bias/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_6/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_6/intermediate/dense/bias/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_6/intermediate/dense/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/intermediate/dense/bias/Assign" op: "Assign" input: "bert/encoder/layer_6/intermediate/dense/bias" input: "bert/encoder/layer_6/intermediate/dense/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/intermediate/dense/bias/read" op: "Identity" input: "bert/encoder/layer_6/intermediate/dense/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_6/intermediate/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_6/intermediate/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_6/intermediate/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_6/intermediate/dense/MatMul" input: "bert/encoder/layer_6/intermediate/dense/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_6/intermediate/dense/Sqrt/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "bert/encoder/layer_6/intermediate/dense/Sqrt" op: "Sqrt" input: "bert/encoder/layer_6/intermediate/dense/Sqrt/x" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/intermediate/dense/truediv" op: "RealDiv" input: "bert/encoder/layer_6/intermediate/dense/BiasAdd" input: "bert/encoder/layer_6/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_6/intermediate/dense/Erf" op: "Erf" input: "bert/encoder/layer_6/intermediate/dense/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_6/intermediate/dense/add/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_6/intermediate/dense/add" op: "Add" input: "bert/encoder/layer_6/intermediate/dense/add/x" input: "bert/encoder/layer_6/intermediate/dense/Erf" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_6/intermediate/dense/mul/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.5 } } } } node { name: "bert/encoder/layer_6/intermediate/dense/mul" op: "Mul" input: "bert/encoder/layer_6/intermediate/dense/mul/x" input: "bert/encoder/layer_6/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_6/intermediate/dense/mul_1" op: "Mul" input: "bert/encoder/layer_6/intermediate/dense/BiasAdd" input: "bert/encoder/layer_6/intermediate/dense/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/output/dense/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/output/dense/kernel/Assign" op: "Assign" input: "bert/encoder/layer_6/output/dense/kernel" input: "bert/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/output/dense/kernel/read" op: "Identity" input: "bert/encoder/layer_6/output/dense/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/output/dense/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/output/dense/bias/Assign" op: "Assign" input: "bert/encoder/layer_6/output/dense/bias" input: "bert/encoder/layer_6/output/dense/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/output/dense/bias/read" op: "Identity" input: "bert/encoder/layer_6/output/dense/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_6/intermediate/dense/mul_1" input: "bert/encoder/layer_6/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_6/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_6/output/dense/MatMul" input: "bert/encoder/layer_6/output/dense/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_6/output/dropout/keep_prob" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "bert/encoder/layer_6/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_6/output/dropout/random_uniform/min" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/output/dropout/random_uniform/max" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_6/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_6/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_6/output/dropout/random_uniform/sub" op: "Sub" input: "bert/encoder/layer_6/output/dropout/random_uniform/max" input: "bert/encoder/layer_6/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/output/dropout/random_uniform/mul" op: "Mul" input: "bert/encoder/layer_6/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_6/output/dropout/random_uniform/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/output/dropout/random_uniform" op: "Add" input: "bert/encoder/layer_6/output/dropout/random_uniform/mul" input: "bert/encoder/layer_6/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/output/dropout/add" op: "Add" input: "bert/encoder/layer_6/output/dropout/keep_prob" input: "bert/encoder/layer_6/output/dropout/random_uniform" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/output/dropout/Floor" op: "Floor" input: "bert/encoder/layer_6/output/dropout/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/output/dropout/div" op: "RealDiv" input: "bert/encoder/layer_6/output/dense/BiasAdd" input: "bert/encoder/layer_6/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/output/dropout/mul" op: "Mul" input: "bert/encoder/layer_6/output/dropout/div" input: "bert/encoder/layer_6/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/output/add" op: "Add" input: "bert/encoder/layer_6/output/dropout/mul" input: "bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/output/LayerNorm/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/output/LayerNorm/beta" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/output/LayerNorm/beta/Assign" op: "Assign" input: "bert/encoder/layer_6/output/LayerNorm/beta" input: "bert/encoder/layer_6/output/LayerNorm/beta/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/output/LayerNorm/beta/read" op: "Identity" input: "bert/encoder/layer_6/output/LayerNorm/beta" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/output/LayerNorm/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_6/output/LayerNorm/gamma" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/output/LayerNorm/gamma/Assign" op: "Assign" input: "bert/encoder/layer_6/output/LayerNorm/gamma" input: "bert/encoder/layer_6/output/LayerNorm/gamma/Initializer/ones" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/output/LayerNorm/gamma/read" op: "Identity" input: "bert/encoder/layer_6/output/LayerNorm/gamma" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/output/LayerNorm/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_6/output/LayerNorm/moments/mean" op: "Mean" input: "bert/encoder/layer_6/output/add" input: "bert/encoder/layer_6/output/LayerNorm/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_6/output/LayerNorm/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_6/output/LayerNorm/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_6/output/LayerNorm/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_6/output/add" input: "bert/encoder/layer_6/output/LayerNorm/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/output/LayerNorm/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_6/output/LayerNorm/moments/variance" op: "Mean" input: "bert/encoder/layer_6/output/LayerNorm/moments/SquaredDifference" input: "bert/encoder/layer_6/output/LayerNorm/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_6/output/LayerNorm/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_6/output/LayerNorm/batchnorm/add" op: "Add" input: "bert/encoder/layer_6/output/LayerNorm/moments/variance" input: "bert/encoder/layer_6/output/LayerNorm/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_6/output/LayerNorm/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_6/output/LayerNorm/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_6/output/LayerNorm/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_6/output/LayerNorm/batchnorm/Rsqrt" input: "bert/encoder/layer_6/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_6/output/add" input: "bert/encoder/layer_6/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_6/output/LayerNorm/moments/mean" input: "bert/encoder/layer_6/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/output/LayerNorm/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_6/output/LayerNorm/beta/read" input: "bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/output/LayerNorm/batchnorm/add_1" op: "Add" input: "bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_1" input: "bert/encoder/layer_6/output/LayerNorm/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/Assign" op: "Assign" input: "bert/encoder/layer_7/attention/self/query/kernel" input: "bert/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/read" op: "Identity" input: "bert/encoder/layer_7/attention/self/query/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/self/query/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/self/query/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/attention/self/query/bias/Assign" op: "Assign" input: "bert/encoder/layer_7/attention/self/query/bias" input: "bert/encoder/layer_7/attention/self/query/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/attention/self/query/bias/read" op: "Identity" input: "bert/encoder/layer_7/attention/self/query/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/self/query/MatMul" op: "MatMul" input: "bert/encoder/layer_6/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_7/attention/self/query/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_7/attention/self/query/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_7/attention/self/query/MatMul" input: "bert/encoder/layer_7/attention/self/query/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/Assign" op: "Assign" input: "bert/encoder/layer_7/attention/self/key/kernel" input: "bert/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/read" op: "Identity" input: "bert/encoder/layer_7/attention/self/key/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/self/key/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/self/key/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/attention/self/key/bias/Assign" op: "Assign" input: "bert/encoder/layer_7/attention/self/key/bias" input: "bert/encoder/layer_7/attention/self/key/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/attention/self/key/bias/read" op: "Identity" input: "bert/encoder/layer_7/attention/self/key/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/self/key/MatMul" op: "MatMul" input: "bert/encoder/layer_6/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_7/attention/self/key/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_7/attention/self/key/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_7/attention/self/key/MatMul" input: "bert/encoder/layer_7/attention/self/key/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/Assign" op: "Assign" input: "bert/encoder/layer_7/attention/self/value/kernel" input: "bert/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/read" op: "Identity" input: "bert/encoder/layer_7/attention/self/value/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/self/value/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/self/value/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/attention/self/value/bias/Assign" op: "Assign" input: "bert/encoder/layer_7/attention/self/value/bias" input: "bert/encoder/layer_7/attention/self/value/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/attention/self/value/bias/read" op: "Identity" input: "bert/encoder/layer_7/attention/self/value/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/self/value/MatMul" op: "MatMul" input: "bert/encoder/layer_6/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_7/attention/self/value/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_7/attention/self/value/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_7/attention/self/value/MatMul" input: "bert/encoder/layer_7/attention/self/value/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_7/attention/self/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_7/attention/self/Reshape" op: "Reshape" input: "bert/encoder/layer_7/attention/self/query/BiasAdd" input: "bert/encoder/layer_7/attention/self/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_7/attention/self/transpose/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_7/attention/self/transpose" op: "Transpose" input: "bert/encoder/layer_7/attention/self/Reshape" input: "bert/encoder/layer_7/attention/self/transpose/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_7/attention/self/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_7/attention/self/Reshape_1" op: "Reshape" input: "bert/encoder/layer_7/attention/self/key/BiasAdd" input: "bert/encoder/layer_7/attention/self/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_7/attention/self/transpose_1/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_7/attention/self/transpose_1" op: "Transpose" input: "bert/encoder/layer_7/attention/self/Reshape_1" input: "bert/encoder/layer_7/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_7/attention/self/MatMul" op: "BatchMatMul" input: "bert/encoder/layer_7/attention/self/transpose" input: "bert/encoder/layer_7/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "bert/encoder/layer_7/attention/self/Mul/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.125 } } } } node { name: "bert/encoder/layer_7/attention/self/Mul" op: "Mul" input: "bert/encoder/layer_7/attention/self/MatMul" input: "bert/encoder/layer_7/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_7/attention/self/ExpandDims/dim" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_7/attention/self/ExpandDims" op: "ExpandDims" input: "bert/encoder/mul" input: "bert/encoder/layer_7/attention/self/ExpandDims/dim" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tdim" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_7/attention/self/sub/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_7/attention/self/sub" op: "Sub" input: "bert/encoder/layer_7/attention/self/sub/x" input: "bert/encoder/layer_7/attention/self/ExpandDims" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_7/attention/self/mul_1/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: -10000.0 } } } } node { name: "bert/encoder/layer_7/attention/self/mul_1" op: "Mul" input: "bert/encoder/layer_7/attention/self/sub" input: "bert/encoder/layer_7/attention/self/mul_1/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_7/attention/self/add" op: "Add" input: "bert/encoder/layer_7/attention/self/Mul" input: "bert/encoder/layer_7/attention/self/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_7/attention/self/Softmax" op: "Softmax" input: "bert/encoder/layer_7/attention/self/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_7/attention/self/dropout/keep_prob" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "bert/encoder/layer_7/attention/self/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "bert/encoder/layer_7/attention/self/dropout/random_uniform/min" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/self/dropout/random_uniform/max" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_7/attention/self/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_7/attention/self/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_7/attention/self/dropout/random_uniform/sub" op: "Sub" input: "bert/encoder/layer_7/attention/self/dropout/random_uniform/max" input: "bert/encoder/layer_7/attention/self/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/attention/self/dropout/random_uniform/mul" op: "Mul" input: "bert/encoder/layer_7/attention/self/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_7/attention/self/dropout/random_uniform/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_7/attention/self/dropout/random_uniform" op: "Add" input: "bert/encoder/layer_7/attention/self/dropout/random_uniform/mul" input: "bert/encoder/layer_7/attention/self/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_7/attention/self/dropout/add" op: "Add" input: "bert/encoder/layer_7/attention/self/dropout/keep_prob" input: "bert/encoder/layer_7/attention/self/dropout/random_uniform" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_7/attention/self/dropout/Floor" op: "Floor" input: "bert/encoder/layer_7/attention/self/dropout/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_7/attention/self/dropout/div" op: "RealDiv" input: "bert/encoder/layer_7/attention/self/Softmax" input: "bert/encoder/layer_7/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_7/attention/self/dropout/mul" op: "Mul" input: "bert/encoder/layer_7/attention/self/dropout/div" input: "bert/encoder/layer_7/attention/self/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_7/attention/self/Reshape_2/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_7/attention/self/Reshape_2" op: "Reshape" input: "bert/encoder/layer_7/attention/self/value/BiasAdd" input: "bert/encoder/layer_7/attention/self/Reshape_2/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_7/attention/self/transpose_2/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_7/attention/self/transpose_2" op: "Transpose" input: "bert/encoder/layer_7/attention/self/Reshape_2" input: "bert/encoder/layer_7/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_7/attention/self/MatMul_1" op: "BatchMatMul" input: "bert/encoder/layer_7/attention/self/dropout/mul" input: "bert/encoder/layer_7/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "bert/encoder/layer_7/attention/self/transpose_3/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_7/attention/self/transpose_3" op: "Transpose" input: "bert/encoder/layer_7/attention/self/MatMul_1" input: "bert/encoder/layer_7/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_7/attention/self/Reshape_3/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_7/attention/self/Reshape_3" op: "Reshape" input: "bert/encoder/layer_7/attention/self/transpose_3" input: "bert/encoder/layer_7/attention/self/Reshape_3/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/Assign" op: "Assign" input: "bert/encoder/layer_7/attention/output/dense/kernel" input: "bert/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/read" op: "Identity" input: "bert/encoder/layer_7/attention/output/dense/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/output/dense/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/attention/output/dense/bias/Assign" op: "Assign" input: "bert/encoder/layer_7/attention/output/dense/bias" input: "bert/encoder/layer_7/attention/output/dense/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/attention/output/dense/bias/read" op: "Identity" input: "bert/encoder/layer_7/attention/output/dense/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_7/attention/self/Reshape_3" input: "bert/encoder/layer_7/attention/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_7/attention/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_7/attention/output/dense/MatMul" input: "bert/encoder/layer_7/attention/output/dense/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_7/attention/output/dropout/keep_prob" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "bert/encoder/layer_7/attention/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_7/attention/output/dropout/random_uniform/min" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/output/dropout/random_uniform/max" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_7/attention/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_7/attention/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_7/attention/output/dropout/random_uniform/sub" op: "Sub" input: "bert/encoder/layer_7/attention/output/dropout/random_uniform/max" input: "bert/encoder/layer_7/attention/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/attention/output/dropout/random_uniform/mul" op: "Mul" input: "bert/encoder/layer_7/attention/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_7/attention/output/dropout/random_uniform/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/output/dropout/random_uniform" op: "Add" input: "bert/encoder/layer_7/attention/output/dropout/random_uniform/mul" input: "bert/encoder/layer_7/attention/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/output/dropout/add" op: "Add" input: "bert/encoder/layer_7/attention/output/dropout/keep_prob" input: "bert/encoder/layer_7/attention/output/dropout/random_uniform" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/output/dropout/Floor" op: "Floor" input: "bert/encoder/layer_7/attention/output/dropout/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/output/dropout/div" op: "RealDiv" input: "bert/encoder/layer_7/attention/output/dense/BiasAdd" input: "bert/encoder/layer_7/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/output/dropout/mul" op: "Mul" input: "bert/encoder/layer_7/attention/output/dropout/div" input: "bert/encoder/layer_7/attention/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/output/add" op: "Add" input: "bert/encoder/layer_7/attention/output/dropout/mul" input: "bert/encoder/layer_6/output/LayerNorm/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/output/LayerNorm/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/output/LayerNorm/beta" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/attention/output/LayerNorm/beta/Assign" op: "Assign" input: "bert/encoder/layer_7/attention/output/LayerNorm/beta" input: "bert/encoder/layer_7/attention/output/LayerNorm/beta/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/attention/output/LayerNorm/beta/read" op: "Identity" input: "bert/encoder/layer_7/attention/output/LayerNorm/beta" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_7/attention/output/LayerNorm/gamma" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/Assign" op: "Assign" input: "bert/encoder/layer_7/attention/output/LayerNorm/gamma" input: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/Initializer/ones" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/read" op: "Identity" input: "bert/encoder/layer_7/attention/output/LayerNorm/gamma" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/output/LayerNorm/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_7/attention/output/LayerNorm/moments/mean" op: "Mean" input: "bert/encoder/layer_7/attention/output/add" input: "bert/encoder/layer_7/attention/output/LayerNorm/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_7/attention/output/LayerNorm/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_7/attention/output/LayerNorm/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_7/attention/output/add" input: "bert/encoder/layer_7/attention/output/LayerNorm/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/output/LayerNorm/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_7/attention/output/LayerNorm/moments/variance" op: "Mean" input: "bert/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference" input: "bert/encoder/layer_7/attention/output/LayerNorm/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/add" op: "Add" input: "bert/encoder/layer_7/attention/output/LayerNorm/moments/variance" input: "bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/Rsqrt" input: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_7/attention/output/add" input: "bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_7/attention/output/LayerNorm/moments/mean" input: "bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_7/attention/output/LayerNorm/beta/read" input: "bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_1" op: "Add" input: "bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_1" input: "bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/Assign" op: "Assign" input: "bert/encoder/layer_7/intermediate/dense/kernel" input: "bert/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/read" op: "Identity" input: "bert/encoder/layer_7/intermediate/dense/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_7/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_7/intermediate/dense/bias/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/intermediate/dense/bias/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_7/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_7/intermediate/dense/bias/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_7/intermediate/dense/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/intermediate/dense/bias/Assign" op: "Assign" input: "bert/encoder/layer_7/intermediate/dense/bias" input: "bert/encoder/layer_7/intermediate/dense/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/intermediate/dense/bias/read" op: "Identity" input: "bert/encoder/layer_7/intermediate/dense/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_7/intermediate/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_7/intermediate/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_7/intermediate/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_7/intermediate/dense/MatMul" input: "bert/encoder/layer_7/intermediate/dense/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_7/intermediate/dense/Sqrt/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "bert/encoder/layer_7/intermediate/dense/Sqrt" op: "Sqrt" input: "bert/encoder/layer_7/intermediate/dense/Sqrt/x" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/intermediate/dense/truediv" op: "RealDiv" input: "bert/encoder/layer_7/intermediate/dense/BiasAdd" input: "bert/encoder/layer_7/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_7/intermediate/dense/Erf" op: "Erf" input: "bert/encoder/layer_7/intermediate/dense/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_7/intermediate/dense/add/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_7/intermediate/dense/add" op: "Add" input: "bert/encoder/layer_7/intermediate/dense/add/x" input: "bert/encoder/layer_7/intermediate/dense/Erf" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_7/intermediate/dense/mul/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.5 } } } } node { name: "bert/encoder/layer_7/intermediate/dense/mul" op: "Mul" input: "bert/encoder/layer_7/intermediate/dense/mul/x" input: "bert/encoder/layer_7/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_7/intermediate/dense/mul_1" op: "Mul" input: "bert/encoder/layer_7/intermediate/dense/BiasAdd" input: "bert/encoder/layer_7/intermediate/dense/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/output/dense/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/output/dense/kernel/Assign" op: "Assign" input: "bert/encoder/layer_7/output/dense/kernel" input: "bert/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/output/dense/kernel/read" op: "Identity" input: "bert/encoder/layer_7/output/dense/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/output/dense/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/output/dense/bias/Assign" op: "Assign" input: "bert/encoder/layer_7/output/dense/bias" input: "bert/encoder/layer_7/output/dense/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/output/dense/bias/read" op: "Identity" input: "bert/encoder/layer_7/output/dense/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_7/intermediate/dense/mul_1" input: "bert/encoder/layer_7/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_7/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_7/output/dense/MatMul" input: "bert/encoder/layer_7/output/dense/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_7/output/dropout/keep_prob" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "bert/encoder/layer_7/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_7/output/dropout/random_uniform/min" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/output/dropout/random_uniform/max" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_7/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_7/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_7/output/dropout/random_uniform/sub" op: "Sub" input: "bert/encoder/layer_7/output/dropout/random_uniform/max" input: "bert/encoder/layer_7/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/output/dropout/random_uniform/mul" op: "Mul" input: "bert/encoder/layer_7/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_7/output/dropout/random_uniform/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/output/dropout/random_uniform" op: "Add" input: "bert/encoder/layer_7/output/dropout/random_uniform/mul" input: "bert/encoder/layer_7/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/output/dropout/add" op: "Add" input: "bert/encoder/layer_7/output/dropout/keep_prob" input: "bert/encoder/layer_7/output/dropout/random_uniform" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/output/dropout/Floor" op: "Floor" input: "bert/encoder/layer_7/output/dropout/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/output/dropout/div" op: "RealDiv" input: "bert/encoder/layer_7/output/dense/BiasAdd" input: "bert/encoder/layer_7/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/output/dropout/mul" op: "Mul" input: "bert/encoder/layer_7/output/dropout/div" input: "bert/encoder/layer_7/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/output/add" op: "Add" input: "bert/encoder/layer_7/output/dropout/mul" input: "bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/output/LayerNorm/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/output/LayerNorm/beta" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/output/LayerNorm/beta/Assign" op: "Assign" input: "bert/encoder/layer_7/output/LayerNorm/beta" input: "bert/encoder/layer_7/output/LayerNorm/beta/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/output/LayerNorm/beta/read" op: "Identity" input: "bert/encoder/layer_7/output/LayerNorm/beta" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/output/LayerNorm/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_7/output/LayerNorm/gamma" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/output/LayerNorm/gamma/Assign" op: "Assign" input: "bert/encoder/layer_7/output/LayerNorm/gamma" input: "bert/encoder/layer_7/output/LayerNorm/gamma/Initializer/ones" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/output/LayerNorm/gamma/read" op: "Identity" input: "bert/encoder/layer_7/output/LayerNorm/gamma" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/output/LayerNorm/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_7/output/LayerNorm/moments/mean" op: "Mean" input: "bert/encoder/layer_7/output/add" input: "bert/encoder/layer_7/output/LayerNorm/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_7/output/LayerNorm/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_7/output/LayerNorm/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_7/output/LayerNorm/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_7/output/add" input: "bert/encoder/layer_7/output/LayerNorm/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/output/LayerNorm/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_7/output/LayerNorm/moments/variance" op: "Mean" input: "bert/encoder/layer_7/output/LayerNorm/moments/SquaredDifference" input: "bert/encoder/layer_7/output/LayerNorm/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_7/output/LayerNorm/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_7/output/LayerNorm/batchnorm/add" op: "Add" input: "bert/encoder/layer_7/output/LayerNorm/moments/variance" input: "bert/encoder/layer_7/output/LayerNorm/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_7/output/LayerNorm/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_7/output/LayerNorm/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_7/output/LayerNorm/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_7/output/LayerNorm/batchnorm/Rsqrt" input: "bert/encoder/layer_7/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_7/output/add" input: "bert/encoder/layer_7/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_7/output/LayerNorm/moments/mean" input: "bert/encoder/layer_7/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/output/LayerNorm/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_7/output/LayerNorm/beta/read" input: "bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/output/LayerNorm/batchnorm/add_1" op: "Add" input: "bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_1" input: "bert/encoder/layer_7/output/LayerNorm/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/Assign" op: "Assign" input: "bert/encoder/layer_8/attention/self/query/kernel" input: "bert/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/read" op: "Identity" input: "bert/encoder/layer_8/attention/self/query/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/self/query/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/self/query/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/attention/self/query/bias/Assign" op: "Assign" input: "bert/encoder/layer_8/attention/self/query/bias" input: "bert/encoder/layer_8/attention/self/query/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/attention/self/query/bias/read" op: "Identity" input: "bert/encoder/layer_8/attention/self/query/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/self/query/MatMul" op: "MatMul" input: "bert/encoder/layer_7/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_8/attention/self/query/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_8/attention/self/query/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_8/attention/self/query/MatMul" input: "bert/encoder/layer_8/attention/self/query/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/Assign" op: "Assign" input: "bert/encoder/layer_8/attention/self/key/kernel" input: "bert/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/read" op: "Identity" input: "bert/encoder/layer_8/attention/self/key/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/self/key/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/self/key/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/attention/self/key/bias/Assign" op: "Assign" input: "bert/encoder/layer_8/attention/self/key/bias" input: "bert/encoder/layer_8/attention/self/key/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/attention/self/key/bias/read" op: "Identity" input: "bert/encoder/layer_8/attention/self/key/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/self/key/MatMul" op: "MatMul" input: "bert/encoder/layer_7/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_8/attention/self/key/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_8/attention/self/key/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_8/attention/self/key/MatMul" input: "bert/encoder/layer_8/attention/self/key/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/Assign" op: "Assign" input: "bert/encoder/layer_8/attention/self/value/kernel" input: "bert/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/read" op: "Identity" input: "bert/encoder/layer_8/attention/self/value/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/self/value/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/self/value/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/attention/self/value/bias/Assign" op: "Assign" input: "bert/encoder/layer_8/attention/self/value/bias" input: "bert/encoder/layer_8/attention/self/value/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/attention/self/value/bias/read" op: "Identity" input: "bert/encoder/layer_8/attention/self/value/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/self/value/MatMul" op: "MatMul" input: "bert/encoder/layer_7/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_8/attention/self/value/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_8/attention/self/value/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_8/attention/self/value/MatMul" input: "bert/encoder/layer_8/attention/self/value/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_8/attention/self/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_8/attention/self/Reshape" op: "Reshape" input: "bert/encoder/layer_8/attention/self/query/BiasAdd" input: "bert/encoder/layer_8/attention/self/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_8/attention/self/transpose/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_8/attention/self/transpose" op: "Transpose" input: "bert/encoder/layer_8/attention/self/Reshape" input: "bert/encoder/layer_8/attention/self/transpose/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_8/attention/self/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_8/attention/self/Reshape_1" op: "Reshape" input: "bert/encoder/layer_8/attention/self/key/BiasAdd" input: "bert/encoder/layer_8/attention/self/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_8/attention/self/transpose_1/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_8/attention/self/transpose_1" op: "Transpose" input: "bert/encoder/layer_8/attention/self/Reshape_1" input: "bert/encoder/layer_8/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_8/attention/self/MatMul" op: "BatchMatMul" input: "bert/encoder/layer_8/attention/self/transpose" input: "bert/encoder/layer_8/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "bert/encoder/layer_8/attention/self/Mul/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.125 } } } } node { name: "bert/encoder/layer_8/attention/self/Mul" op: "Mul" input: "bert/encoder/layer_8/attention/self/MatMul" input: "bert/encoder/layer_8/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_8/attention/self/ExpandDims/dim" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_8/attention/self/ExpandDims" op: "ExpandDims" input: "bert/encoder/mul" input: "bert/encoder/layer_8/attention/self/ExpandDims/dim" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tdim" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_8/attention/self/sub/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_8/attention/self/sub" op: "Sub" input: "bert/encoder/layer_8/attention/self/sub/x" input: "bert/encoder/layer_8/attention/self/ExpandDims" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_8/attention/self/mul_1/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: -10000.0 } } } } node { name: "bert/encoder/layer_8/attention/self/mul_1" op: "Mul" input: "bert/encoder/layer_8/attention/self/sub" input: "bert/encoder/layer_8/attention/self/mul_1/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_8/attention/self/add" op: "Add" input: "bert/encoder/layer_8/attention/self/Mul" input: "bert/encoder/layer_8/attention/self/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_8/attention/self/Softmax" op: "Softmax" input: "bert/encoder/layer_8/attention/self/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_8/attention/self/dropout/keep_prob" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "bert/encoder/layer_8/attention/self/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "bert/encoder/layer_8/attention/self/dropout/random_uniform/min" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/self/dropout/random_uniform/max" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_8/attention/self/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_8/attention/self/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_8/attention/self/dropout/random_uniform/sub" op: "Sub" input: "bert/encoder/layer_8/attention/self/dropout/random_uniform/max" input: "bert/encoder/layer_8/attention/self/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/attention/self/dropout/random_uniform/mul" op: "Mul" input: "bert/encoder/layer_8/attention/self/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_8/attention/self/dropout/random_uniform/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_8/attention/self/dropout/random_uniform" op: "Add" input: "bert/encoder/layer_8/attention/self/dropout/random_uniform/mul" input: "bert/encoder/layer_8/attention/self/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_8/attention/self/dropout/add" op: "Add" input: "bert/encoder/layer_8/attention/self/dropout/keep_prob" input: "bert/encoder/layer_8/attention/self/dropout/random_uniform" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_8/attention/self/dropout/Floor" op: "Floor" input: "bert/encoder/layer_8/attention/self/dropout/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_8/attention/self/dropout/div" op: "RealDiv" input: "bert/encoder/layer_8/attention/self/Softmax" input: "bert/encoder/layer_8/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_8/attention/self/dropout/mul" op: "Mul" input: "bert/encoder/layer_8/attention/self/dropout/div" input: "bert/encoder/layer_8/attention/self/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_8/attention/self/Reshape_2/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_8/attention/self/Reshape_2" op: "Reshape" input: "bert/encoder/layer_8/attention/self/value/BiasAdd" input: "bert/encoder/layer_8/attention/self/Reshape_2/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_8/attention/self/transpose_2/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_8/attention/self/transpose_2" op: "Transpose" input: "bert/encoder/layer_8/attention/self/Reshape_2" input: "bert/encoder/layer_8/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_8/attention/self/MatMul_1" op: "BatchMatMul" input: "bert/encoder/layer_8/attention/self/dropout/mul" input: "bert/encoder/layer_8/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "bert/encoder/layer_8/attention/self/transpose_3/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_8/attention/self/transpose_3" op: "Transpose" input: "bert/encoder/layer_8/attention/self/MatMul_1" input: "bert/encoder/layer_8/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_8/attention/self/Reshape_3/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_8/attention/self/Reshape_3" op: "Reshape" input: "bert/encoder/layer_8/attention/self/transpose_3" input: "bert/encoder/layer_8/attention/self/Reshape_3/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/Assign" op: "Assign" input: "bert/encoder/layer_8/attention/output/dense/kernel" input: "bert/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/read" op: "Identity" input: "bert/encoder/layer_8/attention/output/dense/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/output/dense/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/attention/output/dense/bias/Assign" op: "Assign" input: "bert/encoder/layer_8/attention/output/dense/bias" input: "bert/encoder/layer_8/attention/output/dense/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/attention/output/dense/bias/read" op: "Identity" input: "bert/encoder/layer_8/attention/output/dense/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_8/attention/self/Reshape_3" input: "bert/encoder/layer_8/attention/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_8/attention/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_8/attention/output/dense/MatMul" input: "bert/encoder/layer_8/attention/output/dense/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_8/attention/output/dropout/keep_prob" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "bert/encoder/layer_8/attention/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_8/attention/output/dropout/random_uniform/min" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/output/dropout/random_uniform/max" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_8/attention/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_8/attention/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_8/attention/output/dropout/random_uniform/sub" op: "Sub" input: "bert/encoder/layer_8/attention/output/dropout/random_uniform/max" input: "bert/encoder/layer_8/attention/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/attention/output/dropout/random_uniform/mul" op: "Mul" input: "bert/encoder/layer_8/attention/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_8/attention/output/dropout/random_uniform/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/output/dropout/random_uniform" op: "Add" input: "bert/encoder/layer_8/attention/output/dropout/random_uniform/mul" input: "bert/encoder/layer_8/attention/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/output/dropout/add" op: "Add" input: "bert/encoder/layer_8/attention/output/dropout/keep_prob" input: "bert/encoder/layer_8/attention/output/dropout/random_uniform" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/output/dropout/Floor" op: "Floor" input: "bert/encoder/layer_8/attention/output/dropout/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/output/dropout/div" op: "RealDiv" input: "bert/encoder/layer_8/attention/output/dense/BiasAdd" input: "bert/encoder/layer_8/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/output/dropout/mul" op: "Mul" input: "bert/encoder/layer_8/attention/output/dropout/div" input: "bert/encoder/layer_8/attention/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/output/add" op: "Add" input: "bert/encoder/layer_8/attention/output/dropout/mul" input: "bert/encoder/layer_7/output/LayerNorm/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/output/LayerNorm/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/output/LayerNorm/beta" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/attention/output/LayerNorm/beta/Assign" op: "Assign" input: "bert/encoder/layer_8/attention/output/LayerNorm/beta" input: "bert/encoder/layer_8/attention/output/LayerNorm/beta/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/attention/output/LayerNorm/beta/read" op: "Identity" input: "bert/encoder/layer_8/attention/output/LayerNorm/beta" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/output/LayerNorm/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_8/attention/output/LayerNorm/gamma" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/attention/output/LayerNorm/gamma/Assign" op: "Assign" input: "bert/encoder/layer_8/attention/output/LayerNorm/gamma" input: "bert/encoder/layer_8/attention/output/LayerNorm/gamma/Initializer/ones" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/attention/output/LayerNorm/gamma/read" op: "Identity" input: "bert/encoder/layer_8/attention/output/LayerNorm/gamma" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/output/LayerNorm/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_8/attention/output/LayerNorm/moments/mean" op: "Mean" input: "bert/encoder/layer_8/attention/output/add" input: "bert/encoder/layer_8/attention/output/LayerNorm/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_8/attention/output/LayerNorm/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_8/attention/output/LayerNorm/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_8/attention/output/add" input: "bert/encoder/layer_8/attention/output/LayerNorm/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/output/LayerNorm/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_8/attention/output/LayerNorm/moments/variance" op: "Mean" input: "bert/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference" input: "bert/encoder/layer_8/attention/output/LayerNorm/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/add" op: "Add" input: "bert/encoder/layer_8/attention/output/LayerNorm/moments/variance" input: "bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/Rsqrt" input: "bert/encoder/layer_8/attention/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_8/attention/output/add" input: "bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_8/attention/output/LayerNorm/moments/mean" input: "bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_8/attention/output/LayerNorm/beta/read" input: "bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_1" op: "Add" input: "bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_1" input: "bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/Assign" op: "Assign" input: "bert/encoder/layer_8/intermediate/dense/kernel" input: "bert/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/read" op: "Identity" input: "bert/encoder/layer_8/intermediate/dense/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_8/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_8/intermediate/dense/bias/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/intermediate/dense/bias/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_8/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_8/intermediate/dense/bias/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_8/intermediate/dense/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/intermediate/dense/bias/Assign" op: "Assign" input: "bert/encoder/layer_8/intermediate/dense/bias" input: "bert/encoder/layer_8/intermediate/dense/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/intermediate/dense/bias/read" op: "Identity" input: "bert/encoder/layer_8/intermediate/dense/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_8/intermediate/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_8/intermediate/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_8/intermediate/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_8/intermediate/dense/MatMul" input: "bert/encoder/layer_8/intermediate/dense/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_8/intermediate/dense/Sqrt/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "bert/encoder/layer_8/intermediate/dense/Sqrt" op: "Sqrt" input: "bert/encoder/layer_8/intermediate/dense/Sqrt/x" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/intermediate/dense/truediv" op: "RealDiv" input: "bert/encoder/layer_8/intermediate/dense/BiasAdd" input: "bert/encoder/layer_8/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_8/intermediate/dense/Erf" op: "Erf" input: "bert/encoder/layer_8/intermediate/dense/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_8/intermediate/dense/add/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_8/intermediate/dense/add" op: "Add" input: "bert/encoder/layer_8/intermediate/dense/add/x" input: "bert/encoder/layer_8/intermediate/dense/Erf" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_8/intermediate/dense/mul/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.5 } } } } node { name: "bert/encoder/layer_8/intermediate/dense/mul" op: "Mul" input: "bert/encoder/layer_8/intermediate/dense/mul/x" input: "bert/encoder/layer_8/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_8/intermediate/dense/mul_1" op: "Mul" input: "bert/encoder/layer_8/intermediate/dense/BiasAdd" input: "bert/encoder/layer_8/intermediate/dense/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/output/dense/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/output/dense/kernel/Assign" op: "Assign" input: "bert/encoder/layer_8/output/dense/kernel" input: "bert/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/output/dense/kernel/read" op: "Identity" input: "bert/encoder/layer_8/output/dense/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/output/dense/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/output/dense/bias/Assign" op: "Assign" input: "bert/encoder/layer_8/output/dense/bias" input: "bert/encoder/layer_8/output/dense/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/output/dense/bias/read" op: "Identity" input: "bert/encoder/layer_8/output/dense/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_8/intermediate/dense/mul_1" input: "bert/encoder/layer_8/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_8/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_8/output/dense/MatMul" input: "bert/encoder/layer_8/output/dense/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_8/output/dropout/keep_prob" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "bert/encoder/layer_8/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_8/output/dropout/random_uniform/min" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/output/dropout/random_uniform/max" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_8/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_8/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_8/output/dropout/random_uniform/sub" op: "Sub" input: "bert/encoder/layer_8/output/dropout/random_uniform/max" input: "bert/encoder/layer_8/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/output/dropout/random_uniform/mul" op: "Mul" input: "bert/encoder/layer_8/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_8/output/dropout/random_uniform/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/output/dropout/random_uniform" op: "Add" input: "bert/encoder/layer_8/output/dropout/random_uniform/mul" input: "bert/encoder/layer_8/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/output/dropout/add" op: "Add" input: "bert/encoder/layer_8/output/dropout/keep_prob" input: "bert/encoder/layer_8/output/dropout/random_uniform" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/output/dropout/Floor" op: "Floor" input: "bert/encoder/layer_8/output/dropout/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/output/dropout/div" op: "RealDiv" input: "bert/encoder/layer_8/output/dense/BiasAdd" input: "bert/encoder/layer_8/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/output/dropout/mul" op: "Mul" input: "bert/encoder/layer_8/output/dropout/div" input: "bert/encoder/layer_8/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/output/add" op: "Add" input: "bert/encoder/layer_8/output/dropout/mul" input: "bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/output/LayerNorm/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/output/LayerNorm/beta" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/output/LayerNorm/beta/Assign" op: "Assign" input: "bert/encoder/layer_8/output/LayerNorm/beta" input: "bert/encoder/layer_8/output/LayerNorm/beta/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/output/LayerNorm/beta/read" op: "Identity" input: "bert/encoder/layer_8/output/LayerNorm/beta" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/output/LayerNorm/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_8/output/LayerNorm/gamma" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/output/LayerNorm/gamma/Assign" op: "Assign" input: "bert/encoder/layer_8/output/LayerNorm/gamma" input: "bert/encoder/layer_8/output/LayerNorm/gamma/Initializer/ones" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/output/LayerNorm/gamma/read" op: "Identity" input: "bert/encoder/layer_8/output/LayerNorm/gamma" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/output/LayerNorm/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_8/output/LayerNorm/moments/mean" op: "Mean" input: "bert/encoder/layer_8/output/add" input: "bert/encoder/layer_8/output/LayerNorm/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_8/output/LayerNorm/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_8/output/LayerNorm/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_8/output/LayerNorm/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_8/output/add" input: "bert/encoder/layer_8/output/LayerNorm/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/output/LayerNorm/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_8/output/LayerNorm/moments/variance" op: "Mean" input: "bert/encoder/layer_8/output/LayerNorm/moments/SquaredDifference" input: "bert/encoder/layer_8/output/LayerNorm/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_8/output/LayerNorm/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_8/output/LayerNorm/batchnorm/add" op: "Add" input: "bert/encoder/layer_8/output/LayerNorm/moments/variance" input: "bert/encoder/layer_8/output/LayerNorm/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_8/output/LayerNorm/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_8/output/LayerNorm/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_8/output/LayerNorm/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_8/output/LayerNorm/batchnorm/Rsqrt" input: "bert/encoder/layer_8/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_8/output/add" input: "bert/encoder/layer_8/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_8/output/LayerNorm/moments/mean" input: "bert/encoder/layer_8/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/output/LayerNorm/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_8/output/LayerNorm/beta/read" input: "bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/output/LayerNorm/batchnorm/add_1" op: "Add" input: "bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_1" input: "bert/encoder/layer_8/output/LayerNorm/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/Assign" op: "Assign" input: "bert/encoder/layer_9/attention/self/query/kernel" input: "bert/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/read" op: "Identity" input: "bert/encoder/layer_9/attention/self/query/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/self/query/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/self/query/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/attention/self/query/bias/Assign" op: "Assign" input: "bert/encoder/layer_9/attention/self/query/bias" input: "bert/encoder/layer_9/attention/self/query/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/attention/self/query/bias/read" op: "Identity" input: "bert/encoder/layer_9/attention/self/query/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/self/query/MatMul" op: "MatMul" input: "bert/encoder/layer_8/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_9/attention/self/query/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_9/attention/self/query/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_9/attention/self/query/MatMul" input: "bert/encoder/layer_9/attention/self/query/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/Assign" op: "Assign" input: "bert/encoder/layer_9/attention/self/key/kernel" input: "bert/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/read" op: "Identity" input: "bert/encoder/layer_9/attention/self/key/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/self/key/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/self/key/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/attention/self/key/bias/Assign" op: "Assign" input: "bert/encoder/layer_9/attention/self/key/bias" input: "bert/encoder/layer_9/attention/self/key/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/attention/self/key/bias/read" op: "Identity" input: "bert/encoder/layer_9/attention/self/key/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/self/key/MatMul" op: "MatMul" input: "bert/encoder/layer_8/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_9/attention/self/key/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_9/attention/self/key/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_9/attention/self/key/MatMul" input: "bert/encoder/layer_9/attention/self/key/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/Assign" op: "Assign" input: "bert/encoder/layer_9/attention/self/value/kernel" input: "bert/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/read" op: "Identity" input: "bert/encoder/layer_9/attention/self/value/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/self/value/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/self/value/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/attention/self/value/bias/Assign" op: "Assign" input: "bert/encoder/layer_9/attention/self/value/bias" input: "bert/encoder/layer_9/attention/self/value/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/attention/self/value/bias/read" op: "Identity" input: "bert/encoder/layer_9/attention/self/value/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/self/value/MatMul" op: "MatMul" input: "bert/encoder/layer_8/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_9/attention/self/value/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_9/attention/self/value/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_9/attention/self/value/MatMul" input: "bert/encoder/layer_9/attention/self/value/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_9/attention/self/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_9/attention/self/Reshape" op: "Reshape" input: "bert/encoder/layer_9/attention/self/query/BiasAdd" input: "bert/encoder/layer_9/attention/self/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_9/attention/self/transpose/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_9/attention/self/transpose" op: "Transpose" input: "bert/encoder/layer_9/attention/self/Reshape" input: "bert/encoder/layer_9/attention/self/transpose/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_9/attention/self/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_9/attention/self/Reshape_1" op: "Reshape" input: "bert/encoder/layer_9/attention/self/key/BiasAdd" input: "bert/encoder/layer_9/attention/self/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_9/attention/self/transpose_1/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_9/attention/self/transpose_1" op: "Transpose" input: "bert/encoder/layer_9/attention/self/Reshape_1" input: "bert/encoder/layer_9/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_9/attention/self/MatMul" op: "BatchMatMul" input: "bert/encoder/layer_9/attention/self/transpose" input: "bert/encoder/layer_9/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "bert/encoder/layer_9/attention/self/Mul/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.125 } } } } node { name: "bert/encoder/layer_9/attention/self/Mul" op: "Mul" input: "bert/encoder/layer_9/attention/self/MatMul" input: "bert/encoder/layer_9/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_9/attention/self/ExpandDims/dim" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_9/attention/self/ExpandDims" op: "ExpandDims" input: "bert/encoder/mul" input: "bert/encoder/layer_9/attention/self/ExpandDims/dim" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tdim" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_9/attention/self/sub/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_9/attention/self/sub" op: "Sub" input: "bert/encoder/layer_9/attention/self/sub/x" input: "bert/encoder/layer_9/attention/self/ExpandDims" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_9/attention/self/mul_1/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: -10000.0 } } } } node { name: "bert/encoder/layer_9/attention/self/mul_1" op: "Mul" input: "bert/encoder/layer_9/attention/self/sub" input: "bert/encoder/layer_9/attention/self/mul_1/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_9/attention/self/add" op: "Add" input: "bert/encoder/layer_9/attention/self/Mul" input: "bert/encoder/layer_9/attention/self/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_9/attention/self/Softmax" op: "Softmax" input: "bert/encoder/layer_9/attention/self/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_9/attention/self/dropout/keep_prob" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "bert/encoder/layer_9/attention/self/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "bert/encoder/layer_9/attention/self/dropout/random_uniform/min" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/self/dropout/random_uniform/max" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_9/attention/self/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_9/attention/self/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_9/attention/self/dropout/random_uniform/sub" op: "Sub" input: "bert/encoder/layer_9/attention/self/dropout/random_uniform/max" input: "bert/encoder/layer_9/attention/self/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/attention/self/dropout/random_uniform/mul" op: "Mul" input: "bert/encoder/layer_9/attention/self/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_9/attention/self/dropout/random_uniform/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_9/attention/self/dropout/random_uniform" op: "Add" input: "bert/encoder/layer_9/attention/self/dropout/random_uniform/mul" input: "bert/encoder/layer_9/attention/self/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_9/attention/self/dropout/add" op: "Add" input: "bert/encoder/layer_9/attention/self/dropout/keep_prob" input: "bert/encoder/layer_9/attention/self/dropout/random_uniform" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_9/attention/self/dropout/Floor" op: "Floor" input: "bert/encoder/layer_9/attention/self/dropout/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_9/attention/self/dropout/div" op: "RealDiv" input: "bert/encoder/layer_9/attention/self/Softmax" input: "bert/encoder/layer_9/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_9/attention/self/dropout/mul" op: "Mul" input: "bert/encoder/layer_9/attention/self/dropout/div" input: "bert/encoder/layer_9/attention/self/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_9/attention/self/Reshape_2/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_9/attention/self/Reshape_2" op: "Reshape" input: "bert/encoder/layer_9/attention/self/value/BiasAdd" input: "bert/encoder/layer_9/attention/self/Reshape_2/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_9/attention/self/transpose_2/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_9/attention/self/transpose_2" op: "Transpose" input: "bert/encoder/layer_9/attention/self/Reshape_2" input: "bert/encoder/layer_9/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_9/attention/self/MatMul_1" op: "BatchMatMul" input: "bert/encoder/layer_9/attention/self/dropout/mul" input: "bert/encoder/layer_9/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "bert/encoder/layer_9/attention/self/transpose_3/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_9/attention/self/transpose_3" op: "Transpose" input: "bert/encoder/layer_9/attention/self/MatMul_1" input: "bert/encoder/layer_9/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_9/attention/self/Reshape_3/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_9/attention/self/Reshape_3" op: "Reshape" input: "bert/encoder/layer_9/attention/self/transpose_3" input: "bert/encoder/layer_9/attention/self/Reshape_3/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/Assign" op: "Assign" input: "bert/encoder/layer_9/attention/output/dense/kernel" input: "bert/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/read" op: "Identity" input: "bert/encoder/layer_9/attention/output/dense/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/output/dense/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/attention/output/dense/bias/Assign" op: "Assign" input: "bert/encoder/layer_9/attention/output/dense/bias" input: "bert/encoder/layer_9/attention/output/dense/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/attention/output/dense/bias/read" op: "Identity" input: "bert/encoder/layer_9/attention/output/dense/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_9/attention/self/Reshape_3" input: "bert/encoder/layer_9/attention/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_9/attention/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_9/attention/output/dense/MatMul" input: "bert/encoder/layer_9/attention/output/dense/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_9/attention/output/dropout/keep_prob" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "bert/encoder/layer_9/attention/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_9/attention/output/dropout/random_uniform/min" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/output/dropout/random_uniform/max" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_9/attention/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_9/attention/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_9/attention/output/dropout/random_uniform/sub" op: "Sub" input: "bert/encoder/layer_9/attention/output/dropout/random_uniform/max" input: "bert/encoder/layer_9/attention/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/attention/output/dropout/random_uniform/mul" op: "Mul" input: "bert/encoder/layer_9/attention/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_9/attention/output/dropout/random_uniform/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/output/dropout/random_uniform" op: "Add" input: "bert/encoder/layer_9/attention/output/dropout/random_uniform/mul" input: "bert/encoder/layer_9/attention/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/output/dropout/add" op: "Add" input: "bert/encoder/layer_9/attention/output/dropout/keep_prob" input: "bert/encoder/layer_9/attention/output/dropout/random_uniform" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/output/dropout/Floor" op: "Floor" input: "bert/encoder/layer_9/attention/output/dropout/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/output/dropout/div" op: "RealDiv" input: "bert/encoder/layer_9/attention/output/dense/BiasAdd" input: "bert/encoder/layer_9/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/output/dropout/mul" op: "Mul" input: "bert/encoder/layer_9/attention/output/dropout/div" input: "bert/encoder/layer_9/attention/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/output/add" op: "Add" input: "bert/encoder/layer_9/attention/output/dropout/mul" input: "bert/encoder/layer_8/output/LayerNorm/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/output/LayerNorm/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/output/LayerNorm/beta" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/attention/output/LayerNorm/beta/Assign" op: "Assign" input: "bert/encoder/layer_9/attention/output/LayerNorm/beta" input: "bert/encoder/layer_9/attention/output/LayerNorm/beta/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/attention/output/LayerNorm/beta/read" op: "Identity" input: "bert/encoder/layer_9/attention/output/LayerNorm/beta" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/output/LayerNorm/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_9/attention/output/LayerNorm/gamma" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/attention/output/LayerNorm/gamma/Assign" op: "Assign" input: "bert/encoder/layer_9/attention/output/LayerNorm/gamma" input: "bert/encoder/layer_9/attention/output/LayerNorm/gamma/Initializer/ones" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/attention/output/LayerNorm/gamma/read" op: "Identity" input: "bert/encoder/layer_9/attention/output/LayerNorm/gamma" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/output/LayerNorm/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_9/attention/output/LayerNorm/moments/mean" op: "Mean" input: "bert/encoder/layer_9/attention/output/add" input: "bert/encoder/layer_9/attention/output/LayerNorm/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_9/attention/output/LayerNorm/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_9/attention/output/LayerNorm/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_9/attention/output/add" input: "bert/encoder/layer_9/attention/output/LayerNorm/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/output/LayerNorm/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_9/attention/output/LayerNorm/moments/variance" op: "Mean" input: "bert/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference" input: "bert/encoder/layer_9/attention/output/LayerNorm/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/add" op: "Add" input: "bert/encoder/layer_9/attention/output/LayerNorm/moments/variance" input: "bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/Rsqrt" input: "bert/encoder/layer_9/attention/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_9/attention/output/add" input: "bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_9/attention/output/LayerNorm/moments/mean" input: "bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_9/attention/output/LayerNorm/beta/read" input: "bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_1" op: "Add" input: "bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_1" input: "bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/Assign" op: "Assign" input: "bert/encoder/layer_9/intermediate/dense/kernel" input: "bert/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/read" op: "Identity" input: "bert/encoder/layer_9/intermediate/dense/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_9/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_9/intermediate/dense/bias/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/intermediate/dense/bias/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_9/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_9/intermediate/dense/bias/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_9/intermediate/dense/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/intermediate/dense/bias/Assign" op: "Assign" input: "bert/encoder/layer_9/intermediate/dense/bias" input: "bert/encoder/layer_9/intermediate/dense/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/intermediate/dense/bias/read" op: "Identity" input: "bert/encoder/layer_9/intermediate/dense/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_9/intermediate/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_9/intermediate/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_9/intermediate/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_9/intermediate/dense/MatMul" input: "bert/encoder/layer_9/intermediate/dense/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_9/intermediate/dense/Sqrt/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "bert/encoder/layer_9/intermediate/dense/Sqrt" op: "Sqrt" input: "bert/encoder/layer_9/intermediate/dense/Sqrt/x" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/intermediate/dense/truediv" op: "RealDiv" input: "bert/encoder/layer_9/intermediate/dense/BiasAdd" input: "bert/encoder/layer_9/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_9/intermediate/dense/Erf" op: "Erf" input: "bert/encoder/layer_9/intermediate/dense/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_9/intermediate/dense/add/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_9/intermediate/dense/add" op: "Add" input: "bert/encoder/layer_9/intermediate/dense/add/x" input: "bert/encoder/layer_9/intermediate/dense/Erf" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_9/intermediate/dense/mul/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.5 } } } } node { name: "bert/encoder/layer_9/intermediate/dense/mul" op: "Mul" input: "bert/encoder/layer_9/intermediate/dense/mul/x" input: "bert/encoder/layer_9/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_9/intermediate/dense/mul_1" op: "Mul" input: "bert/encoder/layer_9/intermediate/dense/BiasAdd" input: "bert/encoder/layer_9/intermediate/dense/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/output/dense/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/output/dense/kernel/Assign" op: "Assign" input: "bert/encoder/layer_9/output/dense/kernel" input: "bert/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/output/dense/kernel/read" op: "Identity" input: "bert/encoder/layer_9/output/dense/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/output/dense/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/output/dense/bias/Assign" op: "Assign" input: "bert/encoder/layer_9/output/dense/bias" input: "bert/encoder/layer_9/output/dense/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/output/dense/bias/read" op: "Identity" input: "bert/encoder/layer_9/output/dense/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_9/intermediate/dense/mul_1" input: "bert/encoder/layer_9/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_9/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_9/output/dense/MatMul" input: "bert/encoder/layer_9/output/dense/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_9/output/dropout/keep_prob" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "bert/encoder/layer_9/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_9/output/dropout/random_uniform/min" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/output/dropout/random_uniform/max" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_9/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_9/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_9/output/dropout/random_uniform/sub" op: "Sub" input: "bert/encoder/layer_9/output/dropout/random_uniform/max" input: "bert/encoder/layer_9/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/output/dropout/random_uniform/mul" op: "Mul" input: "bert/encoder/layer_9/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_9/output/dropout/random_uniform/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/output/dropout/random_uniform" op: "Add" input: "bert/encoder/layer_9/output/dropout/random_uniform/mul" input: "bert/encoder/layer_9/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/output/dropout/add" op: "Add" input: "bert/encoder/layer_9/output/dropout/keep_prob" input: "bert/encoder/layer_9/output/dropout/random_uniform" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/output/dropout/Floor" op: "Floor" input: "bert/encoder/layer_9/output/dropout/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/output/dropout/div" op: "RealDiv" input: "bert/encoder/layer_9/output/dense/BiasAdd" input: "bert/encoder/layer_9/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/output/dropout/mul" op: "Mul" input: "bert/encoder/layer_9/output/dropout/div" input: "bert/encoder/layer_9/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/output/add" op: "Add" input: "bert/encoder/layer_9/output/dropout/mul" input: "bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/output/LayerNorm/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/output/LayerNorm/beta" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/output/LayerNorm/beta/Assign" op: "Assign" input: "bert/encoder/layer_9/output/LayerNorm/beta" input: "bert/encoder/layer_9/output/LayerNorm/beta/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/output/LayerNorm/beta/read" op: "Identity" input: "bert/encoder/layer_9/output/LayerNorm/beta" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/output/LayerNorm/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_9/output/LayerNorm/gamma" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/output/LayerNorm/gamma/Assign" op: "Assign" input: "bert/encoder/layer_9/output/LayerNorm/gamma" input: "bert/encoder/layer_9/output/LayerNorm/gamma/Initializer/ones" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/output/LayerNorm/gamma/read" op: "Identity" input: "bert/encoder/layer_9/output/LayerNorm/gamma" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/output/LayerNorm/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_9/output/LayerNorm/moments/mean" op: "Mean" input: "bert/encoder/layer_9/output/add" input: "bert/encoder/layer_9/output/LayerNorm/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_9/output/LayerNorm/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_9/output/LayerNorm/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_9/output/LayerNorm/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_9/output/add" input: "bert/encoder/layer_9/output/LayerNorm/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/output/LayerNorm/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_9/output/LayerNorm/moments/variance" op: "Mean" input: "bert/encoder/layer_9/output/LayerNorm/moments/SquaredDifference" input: "bert/encoder/layer_9/output/LayerNorm/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_9/output/LayerNorm/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_9/output/LayerNorm/batchnorm/add" op: "Add" input: "bert/encoder/layer_9/output/LayerNorm/moments/variance" input: "bert/encoder/layer_9/output/LayerNorm/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_9/output/LayerNorm/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_9/output/LayerNorm/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_9/output/LayerNorm/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_9/output/LayerNorm/batchnorm/Rsqrt" input: "bert/encoder/layer_9/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_9/output/add" input: "bert/encoder/layer_9/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_9/output/LayerNorm/moments/mean" input: "bert/encoder/layer_9/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/output/LayerNorm/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_9/output/LayerNorm/beta/read" input: "bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/output/LayerNorm/batchnorm/add_1" op: "Add" input: "bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_1" input: "bert/encoder/layer_9/output/LayerNorm/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/Assign" op: "Assign" input: "bert/encoder/layer_10/attention/self/query/kernel" input: "bert/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/read" op: "Identity" input: "bert/encoder/layer_10/attention/self/query/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/self/query/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/self/query/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/attention/self/query/bias/Assign" op: "Assign" input: "bert/encoder/layer_10/attention/self/query/bias" input: "bert/encoder/layer_10/attention/self/query/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/attention/self/query/bias/read" op: "Identity" input: "bert/encoder/layer_10/attention/self/query/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/self/query/MatMul" op: "MatMul" input: "bert/encoder/layer_9/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_10/attention/self/query/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_10/attention/self/query/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_10/attention/self/query/MatMul" input: "bert/encoder/layer_10/attention/self/query/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/Assign" op: "Assign" input: "bert/encoder/layer_10/attention/self/key/kernel" input: "bert/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/read" op: "Identity" input: "bert/encoder/layer_10/attention/self/key/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/self/key/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/self/key/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/attention/self/key/bias/Assign" op: "Assign" input: "bert/encoder/layer_10/attention/self/key/bias" input: "bert/encoder/layer_10/attention/self/key/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/attention/self/key/bias/read" op: "Identity" input: "bert/encoder/layer_10/attention/self/key/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/self/key/MatMul" op: "MatMul" input: "bert/encoder/layer_9/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_10/attention/self/key/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_10/attention/self/key/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_10/attention/self/key/MatMul" input: "bert/encoder/layer_10/attention/self/key/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/Assign" op: "Assign" input: "bert/encoder/layer_10/attention/self/value/kernel" input: "bert/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/read" op: "Identity" input: "bert/encoder/layer_10/attention/self/value/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/self/value/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/self/value/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/attention/self/value/bias/Assign" op: "Assign" input: "bert/encoder/layer_10/attention/self/value/bias" input: "bert/encoder/layer_10/attention/self/value/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/attention/self/value/bias/read" op: "Identity" input: "bert/encoder/layer_10/attention/self/value/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/self/value/MatMul" op: "MatMul" input: "bert/encoder/layer_9/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_10/attention/self/value/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_10/attention/self/value/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_10/attention/self/value/MatMul" input: "bert/encoder/layer_10/attention/self/value/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_10/attention/self/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_10/attention/self/Reshape" op: "Reshape" input: "bert/encoder/layer_10/attention/self/query/BiasAdd" input: "bert/encoder/layer_10/attention/self/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_10/attention/self/transpose/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_10/attention/self/transpose" op: "Transpose" input: "bert/encoder/layer_10/attention/self/Reshape" input: "bert/encoder/layer_10/attention/self/transpose/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_10/attention/self/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_10/attention/self/Reshape_1" op: "Reshape" input: "bert/encoder/layer_10/attention/self/key/BiasAdd" input: "bert/encoder/layer_10/attention/self/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_10/attention/self/transpose_1/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_10/attention/self/transpose_1" op: "Transpose" input: "bert/encoder/layer_10/attention/self/Reshape_1" input: "bert/encoder/layer_10/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_10/attention/self/MatMul" op: "BatchMatMul" input: "bert/encoder/layer_10/attention/self/transpose" input: "bert/encoder/layer_10/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "bert/encoder/layer_10/attention/self/Mul/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.125 } } } } node { name: "bert/encoder/layer_10/attention/self/Mul" op: "Mul" input: "bert/encoder/layer_10/attention/self/MatMul" input: "bert/encoder/layer_10/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_10/attention/self/ExpandDims/dim" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_10/attention/self/ExpandDims" op: "ExpandDims" input: "bert/encoder/mul" input: "bert/encoder/layer_10/attention/self/ExpandDims/dim" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tdim" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_10/attention/self/sub/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_10/attention/self/sub" op: "Sub" input: "bert/encoder/layer_10/attention/self/sub/x" input: "bert/encoder/layer_10/attention/self/ExpandDims" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_10/attention/self/mul_1/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: -10000.0 } } } } node { name: "bert/encoder/layer_10/attention/self/mul_1" op: "Mul" input: "bert/encoder/layer_10/attention/self/sub" input: "bert/encoder/layer_10/attention/self/mul_1/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_10/attention/self/add" op: "Add" input: "bert/encoder/layer_10/attention/self/Mul" input: "bert/encoder/layer_10/attention/self/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_10/attention/self/Softmax" op: "Softmax" input: "bert/encoder/layer_10/attention/self/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_10/attention/self/dropout/keep_prob" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "bert/encoder/layer_10/attention/self/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "bert/encoder/layer_10/attention/self/dropout/random_uniform/min" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/self/dropout/random_uniform/max" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_10/attention/self/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_10/attention/self/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_10/attention/self/dropout/random_uniform/sub" op: "Sub" input: "bert/encoder/layer_10/attention/self/dropout/random_uniform/max" input: "bert/encoder/layer_10/attention/self/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/attention/self/dropout/random_uniform/mul" op: "Mul" input: "bert/encoder/layer_10/attention/self/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_10/attention/self/dropout/random_uniform/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_10/attention/self/dropout/random_uniform" op: "Add" input: "bert/encoder/layer_10/attention/self/dropout/random_uniform/mul" input: "bert/encoder/layer_10/attention/self/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_10/attention/self/dropout/add" op: "Add" input: "bert/encoder/layer_10/attention/self/dropout/keep_prob" input: "bert/encoder/layer_10/attention/self/dropout/random_uniform" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_10/attention/self/dropout/Floor" op: "Floor" input: "bert/encoder/layer_10/attention/self/dropout/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_10/attention/self/dropout/div" op: "RealDiv" input: "bert/encoder/layer_10/attention/self/Softmax" input: "bert/encoder/layer_10/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_10/attention/self/dropout/mul" op: "Mul" input: "bert/encoder/layer_10/attention/self/dropout/div" input: "bert/encoder/layer_10/attention/self/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_10/attention/self/Reshape_2/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_10/attention/self/Reshape_2" op: "Reshape" input: "bert/encoder/layer_10/attention/self/value/BiasAdd" input: "bert/encoder/layer_10/attention/self/Reshape_2/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_10/attention/self/transpose_2/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_10/attention/self/transpose_2" op: "Transpose" input: "bert/encoder/layer_10/attention/self/Reshape_2" input: "bert/encoder/layer_10/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_10/attention/self/MatMul_1" op: "BatchMatMul" input: "bert/encoder/layer_10/attention/self/dropout/mul" input: "bert/encoder/layer_10/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "bert/encoder/layer_10/attention/self/transpose_3/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_10/attention/self/transpose_3" op: "Transpose" input: "bert/encoder/layer_10/attention/self/MatMul_1" input: "bert/encoder/layer_10/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_10/attention/self/Reshape_3/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_10/attention/self/Reshape_3" op: "Reshape" input: "bert/encoder/layer_10/attention/self/transpose_3" input: "bert/encoder/layer_10/attention/self/Reshape_3/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/Assign" op: "Assign" input: "bert/encoder/layer_10/attention/output/dense/kernel" input: "bert/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/read" op: "Identity" input: "bert/encoder/layer_10/attention/output/dense/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/output/dense/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/attention/output/dense/bias/Assign" op: "Assign" input: "bert/encoder/layer_10/attention/output/dense/bias" input: "bert/encoder/layer_10/attention/output/dense/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/attention/output/dense/bias/read" op: "Identity" input: "bert/encoder/layer_10/attention/output/dense/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_10/attention/self/Reshape_3" input: "bert/encoder/layer_10/attention/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_10/attention/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_10/attention/output/dense/MatMul" input: "bert/encoder/layer_10/attention/output/dense/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_10/attention/output/dropout/keep_prob" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "bert/encoder/layer_10/attention/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_10/attention/output/dropout/random_uniform/min" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/output/dropout/random_uniform/max" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_10/attention/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_10/attention/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_10/attention/output/dropout/random_uniform/sub" op: "Sub" input: "bert/encoder/layer_10/attention/output/dropout/random_uniform/max" input: "bert/encoder/layer_10/attention/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/attention/output/dropout/random_uniform/mul" op: "Mul" input: "bert/encoder/layer_10/attention/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_10/attention/output/dropout/random_uniform/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/output/dropout/random_uniform" op: "Add" input: "bert/encoder/layer_10/attention/output/dropout/random_uniform/mul" input: "bert/encoder/layer_10/attention/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/output/dropout/add" op: "Add" input: "bert/encoder/layer_10/attention/output/dropout/keep_prob" input: "bert/encoder/layer_10/attention/output/dropout/random_uniform" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/output/dropout/Floor" op: "Floor" input: "bert/encoder/layer_10/attention/output/dropout/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/output/dropout/div" op: "RealDiv" input: "bert/encoder/layer_10/attention/output/dense/BiasAdd" input: "bert/encoder/layer_10/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/output/dropout/mul" op: "Mul" input: "bert/encoder/layer_10/attention/output/dropout/div" input: "bert/encoder/layer_10/attention/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/output/add" op: "Add" input: "bert/encoder/layer_10/attention/output/dropout/mul" input: "bert/encoder/layer_9/output/LayerNorm/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/output/LayerNorm/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/output/LayerNorm/beta" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/attention/output/LayerNorm/beta/Assign" op: "Assign" input: "bert/encoder/layer_10/attention/output/LayerNorm/beta" input: "bert/encoder/layer_10/attention/output/LayerNorm/beta/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/attention/output/LayerNorm/beta/read" op: "Identity" input: "bert/encoder/layer_10/attention/output/LayerNorm/beta" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/output/LayerNorm/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_10/attention/output/LayerNorm/gamma" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/attention/output/LayerNorm/gamma/Assign" op: "Assign" input: "bert/encoder/layer_10/attention/output/LayerNorm/gamma" input: "bert/encoder/layer_10/attention/output/LayerNorm/gamma/Initializer/ones" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/attention/output/LayerNorm/gamma/read" op: "Identity" input: "bert/encoder/layer_10/attention/output/LayerNorm/gamma" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/output/LayerNorm/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_10/attention/output/LayerNorm/moments/mean" op: "Mean" input: "bert/encoder/layer_10/attention/output/add" input: "bert/encoder/layer_10/attention/output/LayerNorm/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_10/attention/output/LayerNorm/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_10/attention/output/LayerNorm/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_10/attention/output/add" input: "bert/encoder/layer_10/attention/output/LayerNorm/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/output/LayerNorm/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_10/attention/output/LayerNorm/moments/variance" op: "Mean" input: "bert/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference" input: "bert/encoder/layer_10/attention/output/LayerNorm/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/add" op: "Add" input: "bert/encoder/layer_10/attention/output/LayerNorm/moments/variance" input: "bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/Rsqrt" input: "bert/encoder/layer_10/attention/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_10/attention/output/add" input: "bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_10/attention/output/LayerNorm/moments/mean" input: "bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_10/attention/output/LayerNorm/beta/read" input: "bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_1" op: "Add" input: "bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_1" input: "bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/Assign" op: "Assign" input: "bert/encoder/layer_10/intermediate/dense/kernel" input: "bert/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/read" op: "Identity" input: "bert/encoder/layer_10/intermediate/dense/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_10/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_10/intermediate/dense/bias/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/intermediate/dense/bias/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_10/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_10/intermediate/dense/bias/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_10/intermediate/dense/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/intermediate/dense/bias/Assign" op: "Assign" input: "bert/encoder/layer_10/intermediate/dense/bias" input: "bert/encoder/layer_10/intermediate/dense/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/intermediate/dense/bias/read" op: "Identity" input: "bert/encoder/layer_10/intermediate/dense/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_10/intermediate/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_10/intermediate/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_10/intermediate/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_10/intermediate/dense/MatMul" input: "bert/encoder/layer_10/intermediate/dense/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_10/intermediate/dense/Sqrt/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "bert/encoder/layer_10/intermediate/dense/Sqrt" op: "Sqrt" input: "bert/encoder/layer_10/intermediate/dense/Sqrt/x" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/intermediate/dense/truediv" op: "RealDiv" input: "bert/encoder/layer_10/intermediate/dense/BiasAdd" input: "bert/encoder/layer_10/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_10/intermediate/dense/Erf" op: "Erf" input: "bert/encoder/layer_10/intermediate/dense/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_10/intermediate/dense/add/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_10/intermediate/dense/add" op: "Add" input: "bert/encoder/layer_10/intermediate/dense/add/x" input: "bert/encoder/layer_10/intermediate/dense/Erf" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_10/intermediate/dense/mul/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.5 } } } } node { name: "bert/encoder/layer_10/intermediate/dense/mul" op: "Mul" input: "bert/encoder/layer_10/intermediate/dense/mul/x" input: "bert/encoder/layer_10/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_10/intermediate/dense/mul_1" op: "Mul" input: "bert/encoder/layer_10/intermediate/dense/BiasAdd" input: "bert/encoder/layer_10/intermediate/dense/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/output/dense/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/output/dense/kernel/Assign" op: "Assign" input: "bert/encoder/layer_10/output/dense/kernel" input: "bert/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/output/dense/kernel/read" op: "Identity" input: "bert/encoder/layer_10/output/dense/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/output/dense/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/output/dense/bias/Assign" op: "Assign" input: "bert/encoder/layer_10/output/dense/bias" input: "bert/encoder/layer_10/output/dense/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/output/dense/bias/read" op: "Identity" input: "bert/encoder/layer_10/output/dense/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_10/intermediate/dense/mul_1" input: "bert/encoder/layer_10/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_10/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_10/output/dense/MatMul" input: "bert/encoder/layer_10/output/dense/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_10/output/dropout/keep_prob" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "bert/encoder/layer_10/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_10/output/dropout/random_uniform/min" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/output/dropout/random_uniform/max" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_10/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_10/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_10/output/dropout/random_uniform/sub" op: "Sub" input: "bert/encoder/layer_10/output/dropout/random_uniform/max" input: "bert/encoder/layer_10/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/output/dropout/random_uniform/mul" op: "Mul" input: "bert/encoder/layer_10/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_10/output/dropout/random_uniform/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/output/dropout/random_uniform" op: "Add" input: "bert/encoder/layer_10/output/dropout/random_uniform/mul" input: "bert/encoder/layer_10/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/output/dropout/add" op: "Add" input: "bert/encoder/layer_10/output/dropout/keep_prob" input: "bert/encoder/layer_10/output/dropout/random_uniform" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/output/dropout/Floor" op: "Floor" input: "bert/encoder/layer_10/output/dropout/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/output/dropout/div" op: "RealDiv" input: "bert/encoder/layer_10/output/dense/BiasAdd" input: "bert/encoder/layer_10/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/output/dropout/mul" op: "Mul" input: "bert/encoder/layer_10/output/dropout/div" input: "bert/encoder/layer_10/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/output/add" op: "Add" input: "bert/encoder/layer_10/output/dropout/mul" input: "bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/output/LayerNorm/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/output/LayerNorm/beta" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/output/LayerNorm/beta/Assign" op: "Assign" input: "bert/encoder/layer_10/output/LayerNorm/beta" input: "bert/encoder/layer_10/output/LayerNorm/beta/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/output/LayerNorm/beta/read" op: "Identity" input: "bert/encoder/layer_10/output/LayerNorm/beta" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/output/LayerNorm/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_10/output/LayerNorm/gamma" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/output/LayerNorm/gamma/Assign" op: "Assign" input: "bert/encoder/layer_10/output/LayerNorm/gamma" input: "bert/encoder/layer_10/output/LayerNorm/gamma/Initializer/ones" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/output/LayerNorm/gamma/read" op: "Identity" input: "bert/encoder/layer_10/output/LayerNorm/gamma" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/output/LayerNorm/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_10/output/LayerNorm/moments/mean" op: "Mean" input: "bert/encoder/layer_10/output/add" input: "bert/encoder/layer_10/output/LayerNorm/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_10/output/LayerNorm/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_10/output/LayerNorm/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_10/output/LayerNorm/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_10/output/add" input: "bert/encoder/layer_10/output/LayerNorm/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/output/LayerNorm/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_10/output/LayerNorm/moments/variance" op: "Mean" input: "bert/encoder/layer_10/output/LayerNorm/moments/SquaredDifference" input: "bert/encoder/layer_10/output/LayerNorm/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_10/output/LayerNorm/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_10/output/LayerNorm/batchnorm/add" op: "Add" input: "bert/encoder/layer_10/output/LayerNorm/moments/variance" input: "bert/encoder/layer_10/output/LayerNorm/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_10/output/LayerNorm/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_10/output/LayerNorm/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_10/output/LayerNorm/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_10/output/LayerNorm/batchnorm/Rsqrt" input: "bert/encoder/layer_10/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_10/output/add" input: "bert/encoder/layer_10/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_10/output/LayerNorm/moments/mean" input: "bert/encoder/layer_10/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/output/LayerNorm/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_10/output/LayerNorm/beta/read" input: "bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/output/LayerNorm/batchnorm/add_1" op: "Add" input: "bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_1" input: "bert/encoder/layer_10/output/LayerNorm/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/Assign" op: "Assign" input: "bert/encoder/layer_11/attention/self/query/kernel" input: "bert/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/read" op: "Identity" input: "bert/encoder/layer_11/attention/self/query/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/self/query/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/self/query/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/attention/self/query/bias/Assign" op: "Assign" input: "bert/encoder/layer_11/attention/self/query/bias" input: "bert/encoder/layer_11/attention/self/query/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/attention/self/query/bias/read" op: "Identity" input: "bert/encoder/layer_11/attention/self/query/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/self/query/MatMul" op: "MatMul" input: "bert/encoder/layer_10/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_11/attention/self/query/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_11/attention/self/query/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_11/attention/self/query/MatMul" input: "bert/encoder/layer_11/attention/self/query/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/Assign" op: "Assign" input: "bert/encoder/layer_11/attention/self/key/kernel" input: "bert/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/read" op: "Identity" input: "bert/encoder/layer_11/attention/self/key/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/self/key/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/self/key/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/attention/self/key/bias/Assign" op: "Assign" input: "bert/encoder/layer_11/attention/self/key/bias" input: "bert/encoder/layer_11/attention/self/key/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/attention/self/key/bias/read" op: "Identity" input: "bert/encoder/layer_11/attention/self/key/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/self/key/MatMul" op: "MatMul" input: "bert/encoder/layer_10/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_11/attention/self/key/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_11/attention/self/key/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_11/attention/self/key/MatMul" input: "bert/encoder/layer_11/attention/self/key/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/Assign" op: "Assign" input: "bert/encoder/layer_11/attention/self/value/kernel" input: "bert/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/read" op: "Identity" input: "bert/encoder/layer_11/attention/self/value/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/self/value/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/self/value/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/attention/self/value/bias/Assign" op: "Assign" input: "bert/encoder/layer_11/attention/self/value/bias" input: "bert/encoder/layer_11/attention/self/value/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/attention/self/value/bias/read" op: "Identity" input: "bert/encoder/layer_11/attention/self/value/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/self/value/MatMul" op: "MatMul" input: "bert/encoder/layer_10/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_11/attention/self/value/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_11/attention/self/value/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_11/attention/self/value/MatMul" input: "bert/encoder/layer_11/attention/self/value/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_11/attention/self/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_11/attention/self/Reshape" op: "Reshape" input: "bert/encoder/layer_11/attention/self/query/BiasAdd" input: "bert/encoder/layer_11/attention/self/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_11/attention/self/transpose/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_11/attention/self/transpose" op: "Transpose" input: "bert/encoder/layer_11/attention/self/Reshape" input: "bert/encoder/layer_11/attention/self/transpose/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_11/attention/self/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_11/attention/self/Reshape_1" op: "Reshape" input: "bert/encoder/layer_11/attention/self/key/BiasAdd" input: "bert/encoder/layer_11/attention/self/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_11/attention/self/transpose_1/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_11/attention/self/transpose_1" op: "Transpose" input: "bert/encoder/layer_11/attention/self/Reshape_1" input: "bert/encoder/layer_11/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_11/attention/self/MatMul" op: "BatchMatMul" input: "bert/encoder/layer_11/attention/self/transpose" input: "bert/encoder/layer_11/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "bert/encoder/layer_11/attention/self/Mul/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.125 } } } } node { name: "bert/encoder/layer_11/attention/self/Mul" op: "Mul" input: "bert/encoder/layer_11/attention/self/MatMul" input: "bert/encoder/layer_11/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_11/attention/self/ExpandDims/dim" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_11/attention/self/ExpandDims" op: "ExpandDims" input: "bert/encoder/mul" input: "bert/encoder/layer_11/attention/self/ExpandDims/dim" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tdim" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_11/attention/self/sub/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_11/attention/self/sub" op: "Sub" input: "bert/encoder/layer_11/attention/self/sub/x" input: "bert/encoder/layer_11/attention/self/ExpandDims" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_11/attention/self/mul_1/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: -10000.0 } } } } node { name: "bert/encoder/layer_11/attention/self/mul_1" op: "Mul" input: "bert/encoder/layer_11/attention/self/sub" input: "bert/encoder/layer_11/attention/self/mul_1/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_11/attention/self/add" op: "Add" input: "bert/encoder/layer_11/attention/self/Mul" input: "bert/encoder/layer_11/attention/self/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_11/attention/self/Softmax" op: "Softmax" input: "bert/encoder/layer_11/attention/self/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_11/attention/self/dropout/keep_prob" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "bert/encoder/layer_11/attention/self/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "bert/encoder/layer_11/attention/self/dropout/random_uniform/min" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/self/dropout/random_uniform/max" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_11/attention/self/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_11/attention/self/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_11/attention/self/dropout/random_uniform/sub" op: "Sub" input: "bert/encoder/layer_11/attention/self/dropout/random_uniform/max" input: "bert/encoder/layer_11/attention/self/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/attention/self/dropout/random_uniform/mul" op: "Mul" input: "bert/encoder/layer_11/attention/self/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_11/attention/self/dropout/random_uniform/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_11/attention/self/dropout/random_uniform" op: "Add" input: "bert/encoder/layer_11/attention/self/dropout/random_uniform/mul" input: "bert/encoder/layer_11/attention/self/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_11/attention/self/dropout/add" op: "Add" input: "bert/encoder/layer_11/attention/self/dropout/keep_prob" input: "bert/encoder/layer_11/attention/self/dropout/random_uniform" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_11/attention/self/dropout/Floor" op: "Floor" input: "bert/encoder/layer_11/attention/self/dropout/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_11/attention/self/dropout/div" op: "RealDiv" input: "bert/encoder/layer_11/attention/self/Softmax" input: "bert/encoder/layer_11/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_11/attention/self/dropout/mul" op: "Mul" input: "bert/encoder/layer_11/attention/self/dropout/div" input: "bert/encoder/layer_11/attention/self/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "bert/encoder/layer_11/attention/self/Reshape_2/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_11/attention/self/Reshape_2" op: "Reshape" input: "bert/encoder/layer_11/attention/self/value/BiasAdd" input: "bert/encoder/layer_11/attention/self/Reshape_2/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_11/attention/self/transpose_2/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_11/attention/self/transpose_2" op: "Transpose" input: "bert/encoder/layer_11/attention/self/Reshape_2" input: "bert/encoder/layer_11/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_11/attention/self/MatMul_1" op: "BatchMatMul" input: "bert/encoder/layer_11/attention/self/dropout/mul" input: "bert/encoder/layer_11/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "bert/encoder/layer_11/attention/self/transpose_3/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_11/attention/self/transpose_3" op: "Transpose" input: "bert/encoder/layer_11/attention/self/MatMul_1" input: "bert/encoder/layer_11/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_11/attention/self/Reshape_3/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_11/attention/self/Reshape_3" op: "Reshape" input: "bert/encoder/layer_11/attention/self/transpose_3" input: "bert/encoder/layer_11/attention/self/Reshape_3/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/Assign" op: "Assign" input: "bert/encoder/layer_11/attention/output/dense/kernel" input: "bert/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/read" op: "Identity" input: "bert/encoder/layer_11/attention/output/dense/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/output/dense/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/attention/output/dense/bias/Assign" op: "Assign" input: "bert/encoder/layer_11/attention/output/dense/bias" input: "bert/encoder/layer_11/attention/output/dense/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/attention/output/dense/bias/read" op: "Identity" input: "bert/encoder/layer_11/attention/output/dense/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_11/attention/self/Reshape_3" input: "bert/encoder/layer_11/attention/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_11/attention/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_11/attention/output/dense/MatMul" input: "bert/encoder/layer_11/attention/output/dense/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_11/attention/output/dropout/keep_prob" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "bert/encoder/layer_11/attention/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_11/attention/output/dropout/random_uniform/min" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/output/dropout/random_uniform/max" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_11/attention/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_11/attention/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_11/attention/output/dropout/random_uniform/sub" op: "Sub" input: "bert/encoder/layer_11/attention/output/dropout/random_uniform/max" input: "bert/encoder/layer_11/attention/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/attention/output/dropout/random_uniform/mul" op: "Mul" input: "bert/encoder/layer_11/attention/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_11/attention/output/dropout/random_uniform/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/output/dropout/random_uniform" op: "Add" input: "bert/encoder/layer_11/attention/output/dropout/random_uniform/mul" input: "bert/encoder/layer_11/attention/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/output/dropout/add" op: "Add" input: "bert/encoder/layer_11/attention/output/dropout/keep_prob" input: "bert/encoder/layer_11/attention/output/dropout/random_uniform" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/output/dropout/Floor" op: "Floor" input: "bert/encoder/layer_11/attention/output/dropout/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/output/dropout/div" op: "RealDiv" input: "bert/encoder/layer_11/attention/output/dense/BiasAdd" input: "bert/encoder/layer_11/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/output/dropout/mul" op: "Mul" input: "bert/encoder/layer_11/attention/output/dropout/div" input: "bert/encoder/layer_11/attention/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/output/add" op: "Add" input: "bert/encoder/layer_11/attention/output/dropout/mul" input: "bert/encoder/layer_10/output/LayerNorm/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/output/LayerNorm/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/output/LayerNorm/beta" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/attention/output/LayerNorm/beta/Assign" op: "Assign" input: "bert/encoder/layer_11/attention/output/LayerNorm/beta" input: "bert/encoder/layer_11/attention/output/LayerNorm/beta/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/attention/output/LayerNorm/beta/read" op: "Identity" input: "bert/encoder/layer_11/attention/output/LayerNorm/beta" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/output/LayerNorm/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_11/attention/output/LayerNorm/gamma" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/attention/output/LayerNorm/gamma/Assign" op: "Assign" input: "bert/encoder/layer_11/attention/output/LayerNorm/gamma" input: "bert/encoder/layer_11/attention/output/LayerNorm/gamma/Initializer/ones" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/attention/output/LayerNorm/gamma/read" op: "Identity" input: "bert/encoder/layer_11/attention/output/LayerNorm/gamma" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/output/LayerNorm/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_11/attention/output/LayerNorm/moments/mean" op: "Mean" input: "bert/encoder/layer_11/attention/output/add" input: "bert/encoder/layer_11/attention/output/LayerNorm/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_11/attention/output/LayerNorm/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_11/attention/output/LayerNorm/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_11/attention/output/add" input: "bert/encoder/layer_11/attention/output/LayerNorm/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/output/LayerNorm/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_11/attention/output/LayerNorm/moments/variance" op: "Mean" input: "bert/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference" input: "bert/encoder/layer_11/attention/output/LayerNorm/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/add" op: "Add" input: "bert/encoder/layer_11/attention/output/LayerNorm/moments/variance" input: "bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/Rsqrt" input: "bert/encoder/layer_11/attention/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_11/attention/output/add" input: "bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_11/attention/output/LayerNorm/moments/mean" input: "bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_11/attention/output/LayerNorm/beta/read" input: "bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_1" op: "Add" input: "bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_1" input: "bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/Assign" op: "Assign" input: "bert/encoder/layer_11/intermediate/dense/kernel" input: "bert/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/read" op: "Identity" input: "bert/encoder/layer_11/intermediate/dense/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_11/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_11/intermediate/dense/bias/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/intermediate/dense/bias/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_11/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_11/intermediate/dense/bias/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_11/intermediate/dense/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/intermediate/dense/bias/Assign" op: "Assign" input: "bert/encoder/layer_11/intermediate/dense/bias" input: "bert/encoder/layer_11/intermediate/dense/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/intermediate/dense/bias/read" op: "Identity" input: "bert/encoder/layer_11/intermediate/dense/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_11/intermediate/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/layer_11/intermediate/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_11/intermediate/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_11/intermediate/dense/MatMul" input: "bert/encoder/layer_11/intermediate/dense/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_11/intermediate/dense/Sqrt/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "bert/encoder/layer_11/intermediate/dense/Sqrt" op: "Sqrt" input: "bert/encoder/layer_11/intermediate/dense/Sqrt/x" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/intermediate/dense/truediv" op: "RealDiv" input: "bert/encoder/layer_11/intermediate/dense/BiasAdd" input: "bert/encoder/layer_11/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_11/intermediate/dense/Erf" op: "Erf" input: "bert/encoder/layer_11/intermediate/dense/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_11/intermediate/dense/add/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_11/intermediate/dense/add" op: "Add" input: "bert/encoder/layer_11/intermediate/dense/add/x" input: "bert/encoder/layer_11/intermediate/dense/Erf" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_11/intermediate/dense/mul/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.5 } } } } node { name: "bert/encoder/layer_11/intermediate/dense/mul" op: "Mul" input: "bert/encoder/layer_11/intermediate/dense/mul/x" input: "bert/encoder/layer_11/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_11/intermediate/dense/mul_1" op: "Mul" input: "bert/encoder/layer_11/intermediate/dense/BiasAdd" input: "bert/encoder/layer_11/intermediate/dense/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal" op: "Add" input: "bert/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/output/dense/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/output/dense/kernel/Assign" op: "Assign" input: "bert/encoder/layer_11/output/dense/kernel" input: "bert/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/output/dense/kernel/read" op: "Identity" input: "bert/encoder/layer_11/output/dense/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/output/dense/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/output/dense/bias/Assign" op: "Assign" input: "bert/encoder/layer_11/output/dense/bias" input: "bert/encoder/layer_11/output/dense/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/output/dense/bias/read" op: "Identity" input: "bert/encoder/layer_11/output/dense/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_11/intermediate/dense/mul_1" input: "bert/encoder/layer_11/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_11/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_11/output/dense/MatMul" input: "bert/encoder/layer_11/output/dense/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_11/output/dropout/keep_prob" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "bert/encoder/layer_11/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_11/output/dropout/random_uniform/min" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/output/dropout/random_uniform/max" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_11/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_11/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_11/output/dropout/random_uniform/sub" op: "Sub" input: "bert/encoder/layer_11/output/dropout/random_uniform/max" input: "bert/encoder/layer_11/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/output/dropout/random_uniform/mul" op: "Mul" input: "bert/encoder/layer_11/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_11/output/dropout/random_uniform/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/output/dropout/random_uniform" op: "Add" input: "bert/encoder/layer_11/output/dropout/random_uniform/mul" input: "bert/encoder/layer_11/output/dropout/random_uniform/min" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/output/dropout/add" op: "Add" input: "bert/encoder/layer_11/output/dropout/keep_prob" input: "bert/encoder/layer_11/output/dropout/random_uniform" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/output/dropout/Floor" op: "Floor" input: "bert/encoder/layer_11/output/dropout/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/output/dropout/div" op: "RealDiv" input: "bert/encoder/layer_11/output/dense/BiasAdd" input: "bert/encoder/layer_11/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/output/dropout/mul" op: "Mul" input: "bert/encoder/layer_11/output/dropout/div" input: "bert/encoder/layer_11/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/output/add" op: "Add" input: "bert/encoder/layer_11/output/dropout/mul" input: "bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/output/LayerNorm/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/output/LayerNorm/beta" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/output/LayerNorm/beta/Assign" op: "Assign" input: "bert/encoder/layer_11/output/LayerNorm/beta" input: "bert/encoder/layer_11/output/LayerNorm/beta/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/output/LayerNorm/beta/read" op: "Identity" input: "bert/encoder/layer_11/output/LayerNorm/beta" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/output/LayerNorm/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_11/output/LayerNorm/gamma" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/output/LayerNorm/gamma/Assign" op: "Assign" input: "bert/encoder/layer_11/output/LayerNorm/gamma" input: "bert/encoder/layer_11/output/LayerNorm/gamma/Initializer/ones" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/output/LayerNorm/gamma/read" op: "Identity" input: "bert/encoder/layer_11/output/LayerNorm/gamma" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/output/LayerNorm/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_11/output/LayerNorm/moments/mean" op: "Mean" input: "bert/encoder/layer_11/output/add" input: "bert/encoder/layer_11/output/LayerNorm/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_11/output/LayerNorm/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_11/output/LayerNorm/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_11/output/LayerNorm/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_11/output/add" input: "bert/encoder/layer_11/output/LayerNorm/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/output/LayerNorm/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_11/output/LayerNorm/moments/variance" op: "Mean" input: "bert/encoder/layer_11/output/LayerNorm/moments/SquaredDifference" input: "bert/encoder/layer_11/output/LayerNorm/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_11/output/LayerNorm/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_11/output/LayerNorm/batchnorm/add" op: "Add" input: "bert/encoder/layer_11/output/LayerNorm/moments/variance" input: "bert/encoder/layer_11/output/LayerNorm/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_11/output/LayerNorm/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_11/output/LayerNorm/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_11/output/LayerNorm/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_11/output/LayerNorm/batchnorm/Rsqrt" input: "bert/encoder/layer_11/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_11/output/add" input: "bert/encoder/layer_11/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_11/output/LayerNorm/moments/mean" input: "bert/encoder/layer_11/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/output/LayerNorm/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_11/output/LayerNorm/beta/read" input: "bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/output/LayerNorm/batchnorm/add_1" op: "Add" input: "bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_1" input: "bert/encoder/layer_11/output/LayerNorm/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "bert/encoder/Reshape_2/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: " \000\000\000\200\000\000\000\000\003\000\000" } } } } node { name: "bert/encoder/Reshape_2" op: "Reshape" input: "bert/encoder/layer_0/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/Reshape_2/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "bert/encoder/Reshape_3/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: " \000\000\000\200\000\000\000\000\003\000\000" } } } } node { name: "bert/encoder/Reshape_3" op: "Reshape" input: "bert/encoder/layer_1/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/Reshape_3/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "bert/encoder/Reshape_4/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: " \000\000\000\200\000\000\000\000\003\000\000" } } } } node { name: "bert/encoder/Reshape_4" op: "Reshape" input: "bert/encoder/layer_2/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/Reshape_4/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "bert/encoder/Reshape_5/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: " \000\000\000\200\000\000\000\000\003\000\000" } } } } node { name: "bert/encoder/Reshape_5" op: "Reshape" input: "bert/encoder/layer_3/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/Reshape_5/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "bert/encoder/Reshape_6/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: " \000\000\000\200\000\000\000\000\003\000\000" } } } } node { name: "bert/encoder/Reshape_6" op: "Reshape" input: "bert/encoder/layer_4/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/Reshape_6/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "bert/encoder/Reshape_7/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: " \000\000\000\200\000\000\000\000\003\000\000" } } } } node { name: "bert/encoder/Reshape_7" op: "Reshape" input: "bert/encoder/layer_5/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/Reshape_7/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "bert/encoder/Reshape_8/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: " \000\000\000\200\000\000\000\000\003\000\000" } } } } node { name: "bert/encoder/Reshape_8" op: "Reshape" input: "bert/encoder/layer_6/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/Reshape_8/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "bert/encoder/Reshape_9/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: " \000\000\000\200\000\000\000\000\003\000\000" } } } } node { name: "bert/encoder/Reshape_9" op: "Reshape" input: "bert/encoder/layer_7/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/Reshape_9/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "bert/encoder/Reshape_10/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: " \000\000\000\200\000\000\000\000\003\000\000" } } } } node { name: "bert/encoder/Reshape_10" op: "Reshape" input: "bert/encoder/layer_8/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/Reshape_10/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "bert/encoder/Reshape_11/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: " \000\000\000\200\000\000\000\000\003\000\000" } } } } node { name: "bert/encoder/Reshape_11" op: "Reshape" input: "bert/encoder/layer_9/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/Reshape_11/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "bert/encoder/Reshape_12/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: " \000\000\000\200\000\000\000\000\003\000\000" } } } } node { name: "bert/encoder/Reshape_12" op: "Reshape" input: "bert/encoder/layer_10/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/Reshape_12/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "bert/encoder/Reshape_13/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: " \000\000\000\200\000\000\000\000\003\000\000" } } } } node { name: "bert/encoder/Reshape_13" op: "Reshape" input: "bert/encoder/layer_11/output/LayerNorm/batchnorm/add_1" input: "bert/encoder/Reshape_13/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "bert/pooler/strided_slice/stack" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\000\000\000\000\000\000\000\000\000\000\000\000" } } } } node { name: "bert/pooler/strided_slice/stack_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\000\000\000\000\001\000\000\000\000\000\000\000" } } } } node { name: "bert/pooler/strided_slice/stack_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\001\000\000\000\001\000\000\000\001\000\000\000" } } } } node { name: "bert/pooler/strided_slice" op: "StridedSlice" input: "bert/encoder/Reshape_13" input: "bert/pooler/strided_slice/stack" input: "bert/pooler/strided_slice/stack_1" input: "bert/pooler/strided_slice/stack_2" attr { key: "Index" value { type: DT_INT32 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 768 } } } } } attr { key: "begin_mask" value { i: 5 } } attr { key: "ellipsis_mask" value { i: 0 } } attr { key: "end_mask" value { i: 5 } } attr { key: "new_axis_mask" value { i: 0 } } attr { key: "shrink_axis_mask" value { i: 0 } } } node { name: "bert/pooler/Squeeze" op: "Squeeze" input: "bert/pooler/strided_slice" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 768 } } } } } attr { key: "squeeze_dims" value { list { i: 1 } } } } node { name: "bert/pooler/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/pooler/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/pooler/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/pooler/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/pooler/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/pooler/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/pooler/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/pooler/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/pooler/dense/kernel/Initializer/truncated_normal" op: "Add" input: "bert/pooler/dense/kernel/Initializer/truncated_normal/mul" input: "bert/pooler/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/pooler/dense/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/pooler/dense/kernel/Assign" op: "Assign" input: "bert/pooler/dense/kernel" input: "bert/pooler/dense/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/pooler/dense/kernel/read" op: "Identity" input: "bert/pooler/dense/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/pooler/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/pooler/dense/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/pooler/dense/bias/Assign" op: "Assign" input: "bert/pooler/dense/bias" input: "bert/pooler/dense/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/pooler/dense/bias/read" op: "Identity" input: "bert/pooler/dense/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/pooler/dense/MatMul" op: "MatMul" input: "bert/pooler/Squeeze" input: "bert/pooler/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/pooler/dense/BiasAdd" op: "BiasAdd" input: "bert/pooler/dense/MatMul" input: "bert/pooler/dense/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/pooler/dense/Tanh" op: "Tanh" input: "bert/pooler/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 768 } } } } } } node { name: "range/start" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "range/limit" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 32 } } } } node { name: "range/delta" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "range" op: "Range" input: "range/start" input: "range/limit" input: "range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } } } } } } node { name: "mul/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 128 } } } } node { name: "mul" op: "Mul" input: "range" input: "mul/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } } } } } } node { name: "Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\377\377\377\377\001\000\000\000" } } } } node { name: "Reshape" op: "Reshape" input: "mul" input: "Reshape/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } } } } } } node { name: "add" op: "Add" input: "IteratorGetNext:3" input: "Reshape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 20 } } } } } } node { name: "Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: -1 } } } } node { name: "Reshape_1" op: "Reshape" input: "add" input: "Reshape_1/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } } } } } } node { name: "Reshape_2/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "Reshape_2" op: "Reshape" input: "bert/encoder/Reshape_13" input: "Reshape_2/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "GatherV2/axis" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "GatherV2" op: "GatherV2" input: "Reshape_2" input: "Reshape_1" input: "GatherV2/axis" attr { key: "Taxis" value { type: DT_INT32 } } attr { key: "Tindices" value { type: DT_INT32 } } attr { key: "Tparams" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "cls/predictions/transform/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "cls/predictions/transform/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "cls/predictions/transform/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "cls/predictions/transform/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "cls/predictions/transform/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "cls/predictions/transform/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "cls/predictions/transform/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "cls/predictions/transform/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "cls/predictions/transform/dense/kernel/Initializer/truncated_normal" op: "Add" input: "cls/predictions/transform/dense/kernel/Initializer/truncated_normal/mul" input: "cls/predictions/transform/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "cls/predictions/transform/dense/kernel" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "cls/predictions/transform/dense/kernel/Assign" op: "Assign" input: "cls/predictions/transform/dense/kernel" input: "cls/predictions/transform/dense/kernel/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "cls/predictions/transform/dense/kernel/read" op: "Identity" input: "cls/predictions/transform/dense/kernel" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "cls/predictions/transform/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "cls/predictions/transform/dense/bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "cls/predictions/transform/dense/bias/Assign" op: "Assign" input: "cls/predictions/transform/dense/bias" input: "cls/predictions/transform/dense/bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "cls/predictions/transform/dense/bias/read" op: "Identity" input: "cls/predictions/transform/dense/bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "cls/predictions/transform/dense/MatMul" op: "MatMul" input: "GatherV2" input: "cls/predictions/transform/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "cls/predictions/transform/dense/BiasAdd" op: "BiasAdd" input: "cls/predictions/transform/dense/MatMul" input: "cls/predictions/transform/dense/bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "cls/predictions/transform/dense/Sqrt/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "cls/predictions/transform/dense/Sqrt" op: "Sqrt" input: "cls/predictions/transform/dense/Sqrt/x" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "cls/predictions/transform/dense/truediv" op: "RealDiv" input: "cls/predictions/transform/dense/BiasAdd" input: "cls/predictions/transform/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "cls/predictions/transform/dense/Erf" op: "Erf" input: "cls/predictions/transform/dense/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "cls/predictions/transform/dense/add/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "cls/predictions/transform/dense/add" op: "Add" input: "cls/predictions/transform/dense/add/x" input: "cls/predictions/transform/dense/Erf" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "cls/predictions/transform/dense/mul/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.5 } } } } node { name: "cls/predictions/transform/dense/mul" op: "Mul" input: "cls/predictions/transform/dense/mul/x" input: "cls/predictions/transform/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "cls/predictions/transform/dense/mul_1" op: "Mul" input: "cls/predictions/transform/dense/BiasAdd" input: "cls/predictions/transform/dense/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "cls/predictions/transform/LayerNorm/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "cls/predictions/transform/LayerNorm/beta" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "cls/predictions/transform/LayerNorm/beta/Assign" op: "Assign" input: "cls/predictions/transform/LayerNorm/beta" input: "cls/predictions/transform/LayerNorm/beta/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "cls/predictions/transform/LayerNorm/beta/read" op: "Identity" input: "cls/predictions/transform/LayerNorm/beta" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "cls/predictions/transform/LayerNorm/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "cls/predictions/transform/LayerNorm/gamma" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "cls/predictions/transform/LayerNorm/gamma/Assign" op: "Assign" input: "cls/predictions/transform/LayerNorm/gamma" input: "cls/predictions/transform/LayerNorm/gamma/Initializer/ones" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "cls/predictions/transform/LayerNorm/gamma/read" op: "Identity" input: "cls/predictions/transform/LayerNorm/gamma" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "cls/predictions/transform/LayerNorm/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "cls/predictions/transform/LayerNorm/moments/mean" op: "Mean" input: "cls/predictions/transform/dense/mul_1" input: "cls/predictions/transform/LayerNorm/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "cls/predictions/transform/LayerNorm/moments/StopGradient" op: "StopGradient" input: "cls/predictions/transform/LayerNorm/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 1 } } } } } } node { name: "cls/predictions/transform/LayerNorm/moments/SquaredDifference" op: "SquaredDifference" input: "cls/predictions/transform/dense/mul_1" input: "cls/predictions/transform/LayerNorm/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "cls/predictions/transform/LayerNorm/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "cls/predictions/transform/LayerNorm/moments/variance" op: "Mean" input: "cls/predictions/transform/LayerNorm/moments/SquaredDifference" input: "cls/predictions/transform/LayerNorm/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "cls/predictions/transform/LayerNorm/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "cls/predictions/transform/LayerNorm/batchnorm/add" op: "Add" input: "cls/predictions/transform/LayerNorm/moments/variance" input: "cls/predictions/transform/LayerNorm/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 1 } } } } } } node { name: "cls/predictions/transform/LayerNorm/batchnorm/Rsqrt" op: "Rsqrt" input: "cls/predictions/transform/LayerNorm/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 1 } } } } } } node { name: "cls/predictions/transform/LayerNorm/batchnorm/mul" op: "Mul" input: "cls/predictions/transform/LayerNorm/batchnorm/Rsqrt" input: "cls/predictions/transform/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "cls/predictions/transform/LayerNorm/batchnorm/mul_1" op: "Mul" input: "cls/predictions/transform/dense/mul_1" input: "cls/predictions/transform/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "cls/predictions/transform/LayerNorm/batchnorm/mul_2" op: "Mul" input: "cls/predictions/transform/LayerNorm/moments/mean" input: "cls/predictions/transform/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "cls/predictions/transform/LayerNorm/batchnorm/sub" op: "Sub" input: "cls/predictions/transform/LayerNorm/beta/read" input: "cls/predictions/transform/LayerNorm/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "cls/predictions/transform/LayerNorm/batchnorm/add_1" op: "Add" input: "cls/predictions/transform/LayerNorm/batchnorm/mul_1" input: "cls/predictions/transform/LayerNorm/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "cls/predictions/output_bias/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@cls/predictions/output_bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 28996 } } } } node { name: "cls/predictions/output_bias/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@cls/predictions/output_bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "cls/predictions/output_bias/Initializer/zeros" op: "Fill" input: "cls/predictions/output_bias/Initializer/zeros/shape_as_tensor" input: "cls/predictions/output_bias/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/output_bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "cls/predictions/output_bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@cls/predictions/output_bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 28996 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "cls/predictions/output_bias/Assign" op: "Assign" input: "cls/predictions/output_bias" input: "cls/predictions/output_bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/output_bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "cls/predictions/output_bias/read" op: "Identity" input: "cls/predictions/output_bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/output_bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } } } } } } node { name: "cls/predictions/MatMul" op: "MatMul" input: "cls/predictions/transform/LayerNorm/batchnorm/add_1" input: "bert/embeddings/word_embeddings/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 28996 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "cls/predictions/BiasAdd" op: "BiasAdd" input: "cls/predictions/MatMul" input: "cls/predictions/output_bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 28996 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "cls/predictions/LogSoftmax" op: "LogSoftmax" input: "cls/predictions/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 28996 } } } } } } node { name: "cls/predictions/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: -1 } } } } node { name: "cls/predictions/Reshape" op: "Reshape" input: "IteratorGetNext:2" input: "cls/predictions/Reshape/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } } } } } } node { name: "cls/predictions/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: -1 } } } } node { name: "cls/predictions/Reshape_1" op: "Reshape" input: "IteratorGetNext:4" input: "cls/predictions/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } } } } } } node { name: "cls/predictions/one_hot/on_value" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "cls/predictions/one_hot/off_value" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "cls/predictions/one_hot/depth" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 28996 } } } } node { name: "cls/predictions/one_hot" op: "OneHot" input: "cls/predictions/Reshape" input: "cls/predictions/one_hot/depth" input: "cls/predictions/one_hot/on_value" input: "cls/predictions/one_hot/off_value" attr { key: "T" value { type: DT_FLOAT } } attr { key: "TI" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 28996 } } } } } attr { key: "axis" value { i: -1 } } } node { name: "cls/predictions/mul" op: "Mul" input: "cls/predictions/LogSoftmax" input: "cls/predictions/one_hot" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 28996 } } } } } } node { name: "cls/predictions/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: -1 } } } } node { name: "cls/predictions/Sum" op: "Sum" input: "cls/predictions/mul" input: "cls/predictions/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "cls/predictions/Neg" op: "Neg" input: "cls/predictions/Sum" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } } } } } } node { name: "cls/predictions/mul_1" op: "Mul" input: "cls/predictions/Reshape_1" input: "cls/predictions/Neg" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } } } } } } node { name: "cls/predictions/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "cls/predictions/Sum_1" op: "Sum" input: "cls/predictions/mul_1" input: "cls/predictions/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "cls/predictions/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "cls/predictions/Sum_2" op: "Sum" input: "cls/predictions/Reshape_1" input: "cls/predictions/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "cls/predictions/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999747378752e-06 } } } } node { name: "cls/predictions/add" op: "Add" input: "cls/predictions/Sum_2" input: "cls/predictions/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "cls/predictions/truediv" op: "RealDiv" input: "cls/predictions/Sum_1" input: "cls/predictions/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "cls/seq_relationship/output_weights/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_weights" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\002\000\000\000\000\003\000\000" } } } } node { name: "cls/seq_relationship/output_weights/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_weights" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "cls/seq_relationship/output_weights/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_weights" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "cls/seq_relationship/output_weights/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "cls/seq_relationship/output_weights/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_weights" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "cls/seq_relationship/output_weights/Initializer/truncated_normal/mul" op: "Mul" input: "cls/seq_relationship/output_weights/Initializer/truncated_normal/TruncatedNormal" input: "cls/seq_relationship/output_weights/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_weights" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "cls/seq_relationship/output_weights/Initializer/truncated_normal" op: "Add" input: "cls/seq_relationship/output_weights/Initializer/truncated_normal/mul" input: "cls/seq_relationship/output_weights/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_weights" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "cls/seq_relationship/output_weights" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_weights" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 2 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "cls/seq_relationship/output_weights/Assign" op: "Assign" input: "cls/seq_relationship/output_weights" input: "cls/seq_relationship/output_weights/Initializer/truncated_normal" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_weights" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "cls/seq_relationship/output_weights/read" op: "Identity" input: "cls/seq_relationship/output_weights" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_weights" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "cls/seq_relationship/output_bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 2 } } float_val: 0.0 } } } } node { name: "cls/seq_relationship/output_bias" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 2 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "cls/seq_relationship/output_bias/Assign" op: "Assign" input: "cls/seq_relationship/output_bias" input: "cls/seq_relationship/output_bias/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "cls/seq_relationship/output_bias/read" op: "Identity" input: "cls/seq_relationship/output_bias" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "cls/seq_relationship/MatMul" op: "MatMul" input: "bert/pooler/dense/Tanh" input: "cls/seq_relationship/output_weights/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 2 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "cls/seq_relationship/BiasAdd" op: "BiasAdd" input: "cls/seq_relationship/MatMul" input: "cls/seq_relationship/output_bias/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 2 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "cls/seq_relationship/LogSoftmax" op: "LogSoftmax" input: "cls/seq_relationship/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 2 } } } } } } node { name: "cls/seq_relationship/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: -1 } } } } node { name: "cls/seq_relationship/Reshape" op: "Reshape" input: "IteratorGetNext:5" input: "cls/seq_relationship/Reshape/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } } } } } } node { name: "cls/seq_relationship/one_hot/on_value" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "cls/seq_relationship/one_hot/off_value" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "cls/seq_relationship/one_hot/depth" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "cls/seq_relationship/one_hot" op: "OneHot" input: "cls/seq_relationship/Reshape" input: "cls/seq_relationship/one_hot/depth" input: "cls/seq_relationship/one_hot/on_value" input: "cls/seq_relationship/one_hot/off_value" attr { key: "T" value { type: DT_FLOAT } } attr { key: "TI" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 2 } } } } } attr { key: "axis" value { i: -1 } } } node { name: "cls/seq_relationship/mul" op: "Mul" input: "cls/seq_relationship/one_hot" input: "cls/seq_relationship/LogSoftmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 2 } } } } } } node { name: "cls/seq_relationship/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: -1 } } } } node { name: "cls/seq_relationship/Sum" op: "Sum" input: "cls/seq_relationship/mul" input: "cls/seq_relationship/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "cls/seq_relationship/Neg" op: "Neg" input: "cls/seq_relationship/Sum" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } } } } } } node { name: "cls/seq_relationship/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "cls/seq_relationship/Mean" op: "Mean" input: "cls/seq_relationship/Neg" input: "cls/seq_relationship/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "add_1" op: "Add" input: "cls/predictions/truediv" input: "cls/seq_relationship/Mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "checkpoint_initializer/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/embeddings/LayerNorm/beta" } } } } node { name: "checkpoint_initializer/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer" op: "RestoreV2" input: "checkpoint_initializer/prefix" input: "checkpoint_initializer/tensor_names" input: "checkpoint_initializer/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign" op: "Assign" input: "bert/embeddings/LayerNorm/beta" input: "checkpoint_initializer" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_1/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_1/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/embeddings/LayerNorm/gamma" } } } } node { name: "checkpoint_initializer_1/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_1" op: "RestoreV2" input: "checkpoint_initializer_1/prefix" input: "checkpoint_initializer_1/tensor_names" input: "checkpoint_initializer_1/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_1" op: "Assign" input: "bert/embeddings/LayerNorm/gamma" input: "checkpoint_initializer_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_2/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_2/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/embeddings/position_embeddings" } } } } node { name: "checkpoint_initializer_2/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_2" op: "RestoreV2" input: "checkpoint_initializer_2/prefix" input: "checkpoint_initializer_2/tensor_names" input: "checkpoint_initializer_2/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_2" op: "Assign" input: "bert/embeddings/position_embeddings" input: "checkpoint_initializer_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_3/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_3/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/embeddings/token_type_embeddings" } } } } node { name: "checkpoint_initializer_3/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_3" op: "RestoreV2" input: "checkpoint_initializer_3/prefix" input: "checkpoint_initializer_3/tensor_names" input: "checkpoint_initializer_3/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_3" op: "Assign" input: "bert/embeddings/token_type_embeddings" input: "checkpoint_initializer_3" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_4/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_4/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/embeddings/word_embeddings" } } } } node { name: "checkpoint_initializer_4/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_4" op: "RestoreV2" input: "checkpoint_initializer_4/prefix" input: "checkpoint_initializer_4/tensor_names" input: "checkpoint_initializer_4/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_4" op: "Assign" input: "bert/embeddings/word_embeddings" input: "checkpoint_initializer_4" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_5/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_5/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_0/attention/output/LayerNorm/beta" } } } } node { name: "checkpoint_initializer_5/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_5" op: "RestoreV2" input: "checkpoint_initializer_5/prefix" input: "checkpoint_initializer_5/tensor_names" input: "checkpoint_initializer_5/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_5" op: "Assign" input: "bert/encoder/layer_0/attention/output/LayerNorm/beta" input: "checkpoint_initializer_5" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_6/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_6/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_0/attention/output/LayerNorm/gamma" } } } } node { name: "checkpoint_initializer_6/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_6" op: "RestoreV2" input: "checkpoint_initializer_6/prefix" input: "checkpoint_initializer_6/tensor_names" input: "checkpoint_initializer_6/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_6" op: "Assign" input: "bert/encoder/layer_0/attention/output/LayerNorm/gamma" input: "checkpoint_initializer_6" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_7/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_7/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_0/attention/output/dense/bias" } } } } node { name: "checkpoint_initializer_7/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_7" op: "RestoreV2" input: "checkpoint_initializer_7/prefix" input: "checkpoint_initializer_7/tensor_names" input: "checkpoint_initializer_7/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_7" op: "Assign" input: "bert/encoder/layer_0/attention/output/dense/bias" input: "checkpoint_initializer_7" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_8/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_8/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_0/attention/output/dense/kernel" } } } } node { name: "checkpoint_initializer_8/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_8" op: "RestoreV2" input: "checkpoint_initializer_8/prefix" input: "checkpoint_initializer_8/tensor_names" input: "checkpoint_initializer_8/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_8" op: "Assign" input: "bert/encoder/layer_0/attention/output/dense/kernel" input: "checkpoint_initializer_8" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_9/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_9/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_0/attention/self/key/bias" } } } } node { name: "checkpoint_initializer_9/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_9" op: "RestoreV2" input: "checkpoint_initializer_9/prefix" input: "checkpoint_initializer_9/tensor_names" input: "checkpoint_initializer_9/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_9" op: "Assign" input: "bert/encoder/layer_0/attention/self/key/bias" input: "checkpoint_initializer_9" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_10/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_10/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_0/attention/self/key/kernel" } } } } node { name: "checkpoint_initializer_10/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_10" op: "RestoreV2" input: "checkpoint_initializer_10/prefix" input: "checkpoint_initializer_10/tensor_names" input: "checkpoint_initializer_10/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_10" op: "Assign" input: "bert/encoder/layer_0/attention/self/key/kernel" input: "checkpoint_initializer_10" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_11/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_11/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_0/attention/self/query/bias" } } } } node { name: "checkpoint_initializer_11/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_11" op: "RestoreV2" input: "checkpoint_initializer_11/prefix" input: "checkpoint_initializer_11/tensor_names" input: "checkpoint_initializer_11/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_11" op: "Assign" input: "bert/encoder/layer_0/attention/self/query/bias" input: "checkpoint_initializer_11" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_12/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_12/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_0/attention/self/query/kernel" } } } } node { name: "checkpoint_initializer_12/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_12" op: "RestoreV2" input: "checkpoint_initializer_12/prefix" input: "checkpoint_initializer_12/tensor_names" input: "checkpoint_initializer_12/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_12" op: "Assign" input: "bert/encoder/layer_0/attention/self/query/kernel" input: "checkpoint_initializer_12" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_13/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_13/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_0/attention/self/value/bias" } } } } node { name: "checkpoint_initializer_13/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_13" op: "RestoreV2" input: "checkpoint_initializer_13/prefix" input: "checkpoint_initializer_13/tensor_names" input: "checkpoint_initializer_13/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_13" op: "Assign" input: "bert/encoder/layer_0/attention/self/value/bias" input: "checkpoint_initializer_13" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_14/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_14/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_0/attention/self/value/kernel" } } } } node { name: "checkpoint_initializer_14/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_14" op: "RestoreV2" input: "checkpoint_initializer_14/prefix" input: "checkpoint_initializer_14/tensor_names" input: "checkpoint_initializer_14/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_14" op: "Assign" input: "bert/encoder/layer_0/attention/self/value/kernel" input: "checkpoint_initializer_14" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_15/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_15/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_0/intermediate/dense/bias" } } } } node { name: "checkpoint_initializer_15/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_15" op: "RestoreV2" input: "checkpoint_initializer_15/prefix" input: "checkpoint_initializer_15/tensor_names" input: "checkpoint_initializer_15/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_15" op: "Assign" input: "bert/encoder/layer_0/intermediate/dense/bias" input: "checkpoint_initializer_15" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_16/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_16/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_0/intermediate/dense/kernel" } } } } node { name: "checkpoint_initializer_16/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_16" op: "RestoreV2" input: "checkpoint_initializer_16/prefix" input: "checkpoint_initializer_16/tensor_names" input: "checkpoint_initializer_16/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_16" op: "Assign" input: "bert/encoder/layer_0/intermediate/dense/kernel" input: "checkpoint_initializer_16" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_17/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_17/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_0/output/LayerNorm/beta" } } } } node { name: "checkpoint_initializer_17/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_17" op: "RestoreV2" input: "checkpoint_initializer_17/prefix" input: "checkpoint_initializer_17/tensor_names" input: "checkpoint_initializer_17/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_17" op: "Assign" input: "bert/encoder/layer_0/output/LayerNorm/beta" input: "checkpoint_initializer_17" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_18/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_18/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_0/output/LayerNorm/gamma" } } } } node { name: "checkpoint_initializer_18/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_18" op: "RestoreV2" input: "checkpoint_initializer_18/prefix" input: "checkpoint_initializer_18/tensor_names" input: "checkpoint_initializer_18/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_18" op: "Assign" input: "bert/encoder/layer_0/output/LayerNorm/gamma" input: "checkpoint_initializer_18" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_19/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_19/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_0/output/dense/bias" } } } } node { name: "checkpoint_initializer_19/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_19" op: "RestoreV2" input: "checkpoint_initializer_19/prefix" input: "checkpoint_initializer_19/tensor_names" input: "checkpoint_initializer_19/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_19" op: "Assign" input: "bert/encoder/layer_0/output/dense/bias" input: "checkpoint_initializer_19" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_20/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_20/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_0/output/dense/kernel" } } } } node { name: "checkpoint_initializer_20/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_20" op: "RestoreV2" input: "checkpoint_initializer_20/prefix" input: "checkpoint_initializer_20/tensor_names" input: "checkpoint_initializer_20/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_20" op: "Assign" input: "bert/encoder/layer_0/output/dense/kernel" input: "checkpoint_initializer_20" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_21/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_21/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_1/attention/output/LayerNorm/beta" } } } } node { name: "checkpoint_initializer_21/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_21" op: "RestoreV2" input: "checkpoint_initializer_21/prefix" input: "checkpoint_initializer_21/tensor_names" input: "checkpoint_initializer_21/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_21" op: "Assign" input: "bert/encoder/layer_1/attention/output/LayerNorm/beta" input: "checkpoint_initializer_21" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_22/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_22/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_1/attention/output/LayerNorm/gamma" } } } } node { name: "checkpoint_initializer_22/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_22" op: "RestoreV2" input: "checkpoint_initializer_22/prefix" input: "checkpoint_initializer_22/tensor_names" input: "checkpoint_initializer_22/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_22" op: "Assign" input: "bert/encoder/layer_1/attention/output/LayerNorm/gamma" input: "checkpoint_initializer_22" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_23/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_23/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_1/attention/output/dense/bias" } } } } node { name: "checkpoint_initializer_23/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_23" op: "RestoreV2" input: "checkpoint_initializer_23/prefix" input: "checkpoint_initializer_23/tensor_names" input: "checkpoint_initializer_23/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_23" op: "Assign" input: "bert/encoder/layer_1/attention/output/dense/bias" input: "checkpoint_initializer_23" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_24/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_24/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_1/attention/output/dense/kernel" } } } } node { name: "checkpoint_initializer_24/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_24" op: "RestoreV2" input: "checkpoint_initializer_24/prefix" input: "checkpoint_initializer_24/tensor_names" input: "checkpoint_initializer_24/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_24" op: "Assign" input: "bert/encoder/layer_1/attention/output/dense/kernel" input: "checkpoint_initializer_24" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_25/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_25/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_1/attention/self/key/bias" } } } } node { name: "checkpoint_initializer_25/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_25" op: "RestoreV2" input: "checkpoint_initializer_25/prefix" input: "checkpoint_initializer_25/tensor_names" input: "checkpoint_initializer_25/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_25" op: "Assign" input: "bert/encoder/layer_1/attention/self/key/bias" input: "checkpoint_initializer_25" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_26/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_26/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_1/attention/self/key/kernel" } } } } node { name: "checkpoint_initializer_26/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_26" op: "RestoreV2" input: "checkpoint_initializer_26/prefix" input: "checkpoint_initializer_26/tensor_names" input: "checkpoint_initializer_26/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_26" op: "Assign" input: "bert/encoder/layer_1/attention/self/key/kernel" input: "checkpoint_initializer_26" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_27/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_27/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_1/attention/self/query/bias" } } } } node { name: "checkpoint_initializer_27/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_27" op: "RestoreV2" input: "checkpoint_initializer_27/prefix" input: "checkpoint_initializer_27/tensor_names" input: "checkpoint_initializer_27/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_27" op: "Assign" input: "bert/encoder/layer_1/attention/self/query/bias" input: "checkpoint_initializer_27" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_28/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_28/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_1/attention/self/query/kernel" } } } } node { name: "checkpoint_initializer_28/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_28" op: "RestoreV2" input: "checkpoint_initializer_28/prefix" input: "checkpoint_initializer_28/tensor_names" input: "checkpoint_initializer_28/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_28" op: "Assign" input: "bert/encoder/layer_1/attention/self/query/kernel" input: "checkpoint_initializer_28" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_29/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_29/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_1/attention/self/value/bias" } } } } node { name: "checkpoint_initializer_29/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_29" op: "RestoreV2" input: "checkpoint_initializer_29/prefix" input: "checkpoint_initializer_29/tensor_names" input: "checkpoint_initializer_29/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_29" op: "Assign" input: "bert/encoder/layer_1/attention/self/value/bias" input: "checkpoint_initializer_29" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_30/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_30/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_1/attention/self/value/kernel" } } } } node { name: "checkpoint_initializer_30/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_30" op: "RestoreV2" input: "checkpoint_initializer_30/prefix" input: "checkpoint_initializer_30/tensor_names" input: "checkpoint_initializer_30/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_30" op: "Assign" input: "bert/encoder/layer_1/attention/self/value/kernel" input: "checkpoint_initializer_30" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_31/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_31/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_1/intermediate/dense/bias" } } } } node { name: "checkpoint_initializer_31/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_31" op: "RestoreV2" input: "checkpoint_initializer_31/prefix" input: "checkpoint_initializer_31/tensor_names" input: "checkpoint_initializer_31/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_31" op: "Assign" input: "bert/encoder/layer_1/intermediate/dense/bias" input: "checkpoint_initializer_31" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_32/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_32/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_1/intermediate/dense/kernel" } } } } node { name: "checkpoint_initializer_32/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_32" op: "RestoreV2" input: "checkpoint_initializer_32/prefix" input: "checkpoint_initializer_32/tensor_names" input: "checkpoint_initializer_32/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_32" op: "Assign" input: "bert/encoder/layer_1/intermediate/dense/kernel" input: "checkpoint_initializer_32" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_33/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_33/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_1/output/LayerNorm/beta" } } } } node { name: "checkpoint_initializer_33/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_33" op: "RestoreV2" input: "checkpoint_initializer_33/prefix" input: "checkpoint_initializer_33/tensor_names" input: "checkpoint_initializer_33/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_33" op: "Assign" input: "bert/encoder/layer_1/output/LayerNorm/beta" input: "checkpoint_initializer_33" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_34/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_34/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_1/output/LayerNorm/gamma" } } } } node { name: "checkpoint_initializer_34/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_34" op: "RestoreV2" input: "checkpoint_initializer_34/prefix" input: "checkpoint_initializer_34/tensor_names" input: "checkpoint_initializer_34/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_34" op: "Assign" input: "bert/encoder/layer_1/output/LayerNorm/gamma" input: "checkpoint_initializer_34" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_35/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_35/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_1/output/dense/bias" } } } } node { name: "checkpoint_initializer_35/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_35" op: "RestoreV2" input: "checkpoint_initializer_35/prefix" input: "checkpoint_initializer_35/tensor_names" input: "checkpoint_initializer_35/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_35" op: "Assign" input: "bert/encoder/layer_1/output/dense/bias" input: "checkpoint_initializer_35" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_36/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_36/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_1/output/dense/kernel" } } } } node { name: "checkpoint_initializer_36/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_36" op: "RestoreV2" input: "checkpoint_initializer_36/prefix" input: "checkpoint_initializer_36/tensor_names" input: "checkpoint_initializer_36/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_36" op: "Assign" input: "bert/encoder/layer_1/output/dense/kernel" input: "checkpoint_initializer_36" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_37/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_37/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_10/attention/output/LayerNorm/beta" } } } } node { name: "checkpoint_initializer_37/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_37" op: "RestoreV2" input: "checkpoint_initializer_37/prefix" input: "checkpoint_initializer_37/tensor_names" input: "checkpoint_initializer_37/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_37" op: "Assign" input: "bert/encoder/layer_10/attention/output/LayerNorm/beta" input: "checkpoint_initializer_37" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_38/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_38/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_10/attention/output/LayerNorm/gamma" } } } } node { name: "checkpoint_initializer_38/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_38" op: "RestoreV2" input: "checkpoint_initializer_38/prefix" input: "checkpoint_initializer_38/tensor_names" input: "checkpoint_initializer_38/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_38" op: "Assign" input: "bert/encoder/layer_10/attention/output/LayerNorm/gamma" input: "checkpoint_initializer_38" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_39/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_39/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_10/attention/output/dense/bias" } } } } node { name: "checkpoint_initializer_39/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_39" op: "RestoreV2" input: "checkpoint_initializer_39/prefix" input: "checkpoint_initializer_39/tensor_names" input: "checkpoint_initializer_39/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_39" op: "Assign" input: "bert/encoder/layer_10/attention/output/dense/bias" input: "checkpoint_initializer_39" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_40/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_40/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_10/attention/output/dense/kernel" } } } } node { name: "checkpoint_initializer_40/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_40" op: "RestoreV2" input: "checkpoint_initializer_40/prefix" input: "checkpoint_initializer_40/tensor_names" input: "checkpoint_initializer_40/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_40" op: "Assign" input: "bert/encoder/layer_10/attention/output/dense/kernel" input: "checkpoint_initializer_40" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_41/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_41/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_10/attention/self/key/bias" } } } } node { name: "checkpoint_initializer_41/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_41" op: "RestoreV2" input: "checkpoint_initializer_41/prefix" input: "checkpoint_initializer_41/tensor_names" input: "checkpoint_initializer_41/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_41" op: "Assign" input: "bert/encoder/layer_10/attention/self/key/bias" input: "checkpoint_initializer_41" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_42/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_42/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_10/attention/self/key/kernel" } } } } node { name: "checkpoint_initializer_42/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_42" op: "RestoreV2" input: "checkpoint_initializer_42/prefix" input: "checkpoint_initializer_42/tensor_names" input: "checkpoint_initializer_42/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_42" op: "Assign" input: "bert/encoder/layer_10/attention/self/key/kernel" input: "checkpoint_initializer_42" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_43/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_43/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_10/attention/self/query/bias" } } } } node { name: "checkpoint_initializer_43/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_43" op: "RestoreV2" input: "checkpoint_initializer_43/prefix" input: "checkpoint_initializer_43/tensor_names" input: "checkpoint_initializer_43/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_43" op: "Assign" input: "bert/encoder/layer_10/attention/self/query/bias" input: "checkpoint_initializer_43" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_44/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_44/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_10/attention/self/query/kernel" } } } } node { name: "checkpoint_initializer_44/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_44" op: "RestoreV2" input: "checkpoint_initializer_44/prefix" input: "checkpoint_initializer_44/tensor_names" input: "checkpoint_initializer_44/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_44" op: "Assign" input: "bert/encoder/layer_10/attention/self/query/kernel" input: "checkpoint_initializer_44" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_45/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_45/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_10/attention/self/value/bias" } } } } node { name: "checkpoint_initializer_45/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_45" op: "RestoreV2" input: "checkpoint_initializer_45/prefix" input: "checkpoint_initializer_45/tensor_names" input: "checkpoint_initializer_45/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_45" op: "Assign" input: "bert/encoder/layer_10/attention/self/value/bias" input: "checkpoint_initializer_45" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_46/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_46/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_10/attention/self/value/kernel" } } } } node { name: "checkpoint_initializer_46/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_46" op: "RestoreV2" input: "checkpoint_initializer_46/prefix" input: "checkpoint_initializer_46/tensor_names" input: "checkpoint_initializer_46/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_46" op: "Assign" input: "bert/encoder/layer_10/attention/self/value/kernel" input: "checkpoint_initializer_46" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_47/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_47/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_10/intermediate/dense/bias" } } } } node { name: "checkpoint_initializer_47/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_47" op: "RestoreV2" input: "checkpoint_initializer_47/prefix" input: "checkpoint_initializer_47/tensor_names" input: "checkpoint_initializer_47/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_47" op: "Assign" input: "bert/encoder/layer_10/intermediate/dense/bias" input: "checkpoint_initializer_47" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_48/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_48/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_10/intermediate/dense/kernel" } } } } node { name: "checkpoint_initializer_48/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_48" op: "RestoreV2" input: "checkpoint_initializer_48/prefix" input: "checkpoint_initializer_48/tensor_names" input: "checkpoint_initializer_48/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_48" op: "Assign" input: "bert/encoder/layer_10/intermediate/dense/kernel" input: "checkpoint_initializer_48" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_49/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_49/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_10/output/LayerNorm/beta" } } } } node { name: "checkpoint_initializer_49/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_49" op: "RestoreV2" input: "checkpoint_initializer_49/prefix" input: "checkpoint_initializer_49/tensor_names" input: "checkpoint_initializer_49/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_49" op: "Assign" input: "bert/encoder/layer_10/output/LayerNorm/beta" input: "checkpoint_initializer_49" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_50/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_50/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_10/output/LayerNorm/gamma" } } } } node { name: "checkpoint_initializer_50/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_50" op: "RestoreV2" input: "checkpoint_initializer_50/prefix" input: "checkpoint_initializer_50/tensor_names" input: "checkpoint_initializer_50/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_50" op: "Assign" input: "bert/encoder/layer_10/output/LayerNorm/gamma" input: "checkpoint_initializer_50" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_51/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_51/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_10/output/dense/bias" } } } } node { name: "checkpoint_initializer_51/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_51" op: "RestoreV2" input: "checkpoint_initializer_51/prefix" input: "checkpoint_initializer_51/tensor_names" input: "checkpoint_initializer_51/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_51" op: "Assign" input: "bert/encoder/layer_10/output/dense/bias" input: "checkpoint_initializer_51" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_52/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_52/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_10/output/dense/kernel" } } } } node { name: "checkpoint_initializer_52/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_52" op: "RestoreV2" input: "checkpoint_initializer_52/prefix" input: "checkpoint_initializer_52/tensor_names" input: "checkpoint_initializer_52/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_52" op: "Assign" input: "bert/encoder/layer_10/output/dense/kernel" input: "checkpoint_initializer_52" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_53/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_53/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_11/attention/output/LayerNorm/beta" } } } } node { name: "checkpoint_initializer_53/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_53" op: "RestoreV2" input: "checkpoint_initializer_53/prefix" input: "checkpoint_initializer_53/tensor_names" input: "checkpoint_initializer_53/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_53" op: "Assign" input: "bert/encoder/layer_11/attention/output/LayerNorm/beta" input: "checkpoint_initializer_53" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_54/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_54/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_11/attention/output/LayerNorm/gamma" } } } } node { name: "checkpoint_initializer_54/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_54" op: "RestoreV2" input: "checkpoint_initializer_54/prefix" input: "checkpoint_initializer_54/tensor_names" input: "checkpoint_initializer_54/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_54" op: "Assign" input: "bert/encoder/layer_11/attention/output/LayerNorm/gamma" input: "checkpoint_initializer_54" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_55/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_55/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_11/attention/output/dense/bias" } } } } node { name: "checkpoint_initializer_55/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_55" op: "RestoreV2" input: "checkpoint_initializer_55/prefix" input: "checkpoint_initializer_55/tensor_names" input: "checkpoint_initializer_55/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_55" op: "Assign" input: "bert/encoder/layer_11/attention/output/dense/bias" input: "checkpoint_initializer_55" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_56/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_56/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_11/attention/output/dense/kernel" } } } } node { name: "checkpoint_initializer_56/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_56" op: "RestoreV2" input: "checkpoint_initializer_56/prefix" input: "checkpoint_initializer_56/tensor_names" input: "checkpoint_initializer_56/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_56" op: "Assign" input: "bert/encoder/layer_11/attention/output/dense/kernel" input: "checkpoint_initializer_56" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_57/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_57/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_11/attention/self/key/bias" } } } } node { name: "checkpoint_initializer_57/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_57" op: "RestoreV2" input: "checkpoint_initializer_57/prefix" input: "checkpoint_initializer_57/tensor_names" input: "checkpoint_initializer_57/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_57" op: "Assign" input: "bert/encoder/layer_11/attention/self/key/bias" input: "checkpoint_initializer_57" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_58/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_58/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_11/attention/self/key/kernel" } } } } node { name: "checkpoint_initializer_58/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_58" op: "RestoreV2" input: "checkpoint_initializer_58/prefix" input: "checkpoint_initializer_58/tensor_names" input: "checkpoint_initializer_58/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_58" op: "Assign" input: "bert/encoder/layer_11/attention/self/key/kernel" input: "checkpoint_initializer_58" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_59/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_59/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_11/attention/self/query/bias" } } } } node { name: "checkpoint_initializer_59/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_59" op: "RestoreV2" input: "checkpoint_initializer_59/prefix" input: "checkpoint_initializer_59/tensor_names" input: "checkpoint_initializer_59/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_59" op: "Assign" input: "bert/encoder/layer_11/attention/self/query/bias" input: "checkpoint_initializer_59" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_60/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_60/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_11/attention/self/query/kernel" } } } } node { name: "checkpoint_initializer_60/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_60" op: "RestoreV2" input: "checkpoint_initializer_60/prefix" input: "checkpoint_initializer_60/tensor_names" input: "checkpoint_initializer_60/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_60" op: "Assign" input: "bert/encoder/layer_11/attention/self/query/kernel" input: "checkpoint_initializer_60" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_61/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_61/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_11/attention/self/value/bias" } } } } node { name: "checkpoint_initializer_61/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_61" op: "RestoreV2" input: "checkpoint_initializer_61/prefix" input: "checkpoint_initializer_61/tensor_names" input: "checkpoint_initializer_61/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_61" op: "Assign" input: "bert/encoder/layer_11/attention/self/value/bias" input: "checkpoint_initializer_61" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_62/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_62/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_11/attention/self/value/kernel" } } } } node { name: "checkpoint_initializer_62/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_62" op: "RestoreV2" input: "checkpoint_initializer_62/prefix" input: "checkpoint_initializer_62/tensor_names" input: "checkpoint_initializer_62/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_62" op: "Assign" input: "bert/encoder/layer_11/attention/self/value/kernel" input: "checkpoint_initializer_62" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_63/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_63/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_11/intermediate/dense/bias" } } } } node { name: "checkpoint_initializer_63/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_63" op: "RestoreV2" input: "checkpoint_initializer_63/prefix" input: "checkpoint_initializer_63/tensor_names" input: "checkpoint_initializer_63/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_63" op: "Assign" input: "bert/encoder/layer_11/intermediate/dense/bias" input: "checkpoint_initializer_63" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_64/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_64/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_11/intermediate/dense/kernel" } } } } node { name: "checkpoint_initializer_64/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_64" op: "RestoreV2" input: "checkpoint_initializer_64/prefix" input: "checkpoint_initializer_64/tensor_names" input: "checkpoint_initializer_64/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_64" op: "Assign" input: "bert/encoder/layer_11/intermediate/dense/kernel" input: "checkpoint_initializer_64" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_65/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_65/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_11/output/LayerNorm/beta" } } } } node { name: "checkpoint_initializer_65/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_65" op: "RestoreV2" input: "checkpoint_initializer_65/prefix" input: "checkpoint_initializer_65/tensor_names" input: "checkpoint_initializer_65/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_65" op: "Assign" input: "bert/encoder/layer_11/output/LayerNorm/beta" input: "checkpoint_initializer_65" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_66/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_66/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_11/output/LayerNorm/gamma" } } } } node { name: "checkpoint_initializer_66/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_66" op: "RestoreV2" input: "checkpoint_initializer_66/prefix" input: "checkpoint_initializer_66/tensor_names" input: "checkpoint_initializer_66/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_66" op: "Assign" input: "bert/encoder/layer_11/output/LayerNorm/gamma" input: "checkpoint_initializer_66" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_67/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_67/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_11/output/dense/bias" } } } } node { name: "checkpoint_initializer_67/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_67" op: "RestoreV2" input: "checkpoint_initializer_67/prefix" input: "checkpoint_initializer_67/tensor_names" input: "checkpoint_initializer_67/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_67" op: "Assign" input: "bert/encoder/layer_11/output/dense/bias" input: "checkpoint_initializer_67" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_68/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_68/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_11/output/dense/kernel" } } } } node { name: "checkpoint_initializer_68/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_68" op: "RestoreV2" input: "checkpoint_initializer_68/prefix" input: "checkpoint_initializer_68/tensor_names" input: "checkpoint_initializer_68/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_68" op: "Assign" input: "bert/encoder/layer_11/output/dense/kernel" input: "checkpoint_initializer_68" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_69/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_69/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_2/attention/output/LayerNorm/beta" } } } } node { name: "checkpoint_initializer_69/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_69" op: "RestoreV2" input: "checkpoint_initializer_69/prefix" input: "checkpoint_initializer_69/tensor_names" input: "checkpoint_initializer_69/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_69" op: "Assign" input: "bert/encoder/layer_2/attention/output/LayerNorm/beta" input: "checkpoint_initializer_69" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_70/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_70/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_2/attention/output/LayerNorm/gamma" } } } } node { name: "checkpoint_initializer_70/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_70" op: "RestoreV2" input: "checkpoint_initializer_70/prefix" input: "checkpoint_initializer_70/tensor_names" input: "checkpoint_initializer_70/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_70" op: "Assign" input: "bert/encoder/layer_2/attention/output/LayerNorm/gamma" input: "checkpoint_initializer_70" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_71/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_71/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_2/attention/output/dense/bias" } } } } node { name: "checkpoint_initializer_71/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_71" op: "RestoreV2" input: "checkpoint_initializer_71/prefix" input: "checkpoint_initializer_71/tensor_names" input: "checkpoint_initializer_71/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_71" op: "Assign" input: "bert/encoder/layer_2/attention/output/dense/bias" input: "checkpoint_initializer_71" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_72/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_72/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_2/attention/output/dense/kernel" } } } } node { name: "checkpoint_initializer_72/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_72" op: "RestoreV2" input: "checkpoint_initializer_72/prefix" input: "checkpoint_initializer_72/tensor_names" input: "checkpoint_initializer_72/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_72" op: "Assign" input: "bert/encoder/layer_2/attention/output/dense/kernel" input: "checkpoint_initializer_72" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_73/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_73/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_2/attention/self/key/bias" } } } } node { name: "checkpoint_initializer_73/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_73" op: "RestoreV2" input: "checkpoint_initializer_73/prefix" input: "checkpoint_initializer_73/tensor_names" input: "checkpoint_initializer_73/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_73" op: "Assign" input: "bert/encoder/layer_2/attention/self/key/bias" input: "checkpoint_initializer_73" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_74/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_74/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_2/attention/self/key/kernel" } } } } node { name: "checkpoint_initializer_74/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_74" op: "RestoreV2" input: "checkpoint_initializer_74/prefix" input: "checkpoint_initializer_74/tensor_names" input: "checkpoint_initializer_74/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_74" op: "Assign" input: "bert/encoder/layer_2/attention/self/key/kernel" input: "checkpoint_initializer_74" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_75/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_75/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_2/attention/self/query/bias" } } } } node { name: "checkpoint_initializer_75/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_75" op: "RestoreV2" input: "checkpoint_initializer_75/prefix" input: "checkpoint_initializer_75/tensor_names" input: "checkpoint_initializer_75/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_75" op: "Assign" input: "bert/encoder/layer_2/attention/self/query/bias" input: "checkpoint_initializer_75" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_76/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_76/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_2/attention/self/query/kernel" } } } } node { name: "checkpoint_initializer_76/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_76" op: "RestoreV2" input: "checkpoint_initializer_76/prefix" input: "checkpoint_initializer_76/tensor_names" input: "checkpoint_initializer_76/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_76" op: "Assign" input: "bert/encoder/layer_2/attention/self/query/kernel" input: "checkpoint_initializer_76" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_77/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_77/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_2/attention/self/value/bias" } } } } node { name: "checkpoint_initializer_77/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_77" op: "RestoreV2" input: "checkpoint_initializer_77/prefix" input: "checkpoint_initializer_77/tensor_names" input: "checkpoint_initializer_77/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_77" op: "Assign" input: "bert/encoder/layer_2/attention/self/value/bias" input: "checkpoint_initializer_77" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_78/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_78/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_2/attention/self/value/kernel" } } } } node { name: "checkpoint_initializer_78/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_78" op: "RestoreV2" input: "checkpoint_initializer_78/prefix" input: "checkpoint_initializer_78/tensor_names" input: "checkpoint_initializer_78/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_78" op: "Assign" input: "bert/encoder/layer_2/attention/self/value/kernel" input: "checkpoint_initializer_78" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_79/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_79/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_2/intermediate/dense/bias" } } } } node { name: "checkpoint_initializer_79/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_79" op: "RestoreV2" input: "checkpoint_initializer_79/prefix" input: "checkpoint_initializer_79/tensor_names" input: "checkpoint_initializer_79/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_79" op: "Assign" input: "bert/encoder/layer_2/intermediate/dense/bias" input: "checkpoint_initializer_79" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_80/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_80/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_2/intermediate/dense/kernel" } } } } node { name: "checkpoint_initializer_80/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_80" op: "RestoreV2" input: "checkpoint_initializer_80/prefix" input: "checkpoint_initializer_80/tensor_names" input: "checkpoint_initializer_80/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_80" op: "Assign" input: "bert/encoder/layer_2/intermediate/dense/kernel" input: "checkpoint_initializer_80" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_81/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_81/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_2/output/LayerNorm/beta" } } } } node { name: "checkpoint_initializer_81/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_81" op: "RestoreV2" input: "checkpoint_initializer_81/prefix" input: "checkpoint_initializer_81/tensor_names" input: "checkpoint_initializer_81/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_81" op: "Assign" input: "bert/encoder/layer_2/output/LayerNorm/beta" input: "checkpoint_initializer_81" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_82/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_82/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_2/output/LayerNorm/gamma" } } } } node { name: "checkpoint_initializer_82/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_82" op: "RestoreV2" input: "checkpoint_initializer_82/prefix" input: "checkpoint_initializer_82/tensor_names" input: "checkpoint_initializer_82/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_82" op: "Assign" input: "bert/encoder/layer_2/output/LayerNorm/gamma" input: "checkpoint_initializer_82" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_83/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_83/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_2/output/dense/bias" } } } } node { name: "checkpoint_initializer_83/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_83" op: "RestoreV2" input: "checkpoint_initializer_83/prefix" input: "checkpoint_initializer_83/tensor_names" input: "checkpoint_initializer_83/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_83" op: "Assign" input: "bert/encoder/layer_2/output/dense/bias" input: "checkpoint_initializer_83" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_84/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_84/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_2/output/dense/kernel" } } } } node { name: "checkpoint_initializer_84/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_84" op: "RestoreV2" input: "checkpoint_initializer_84/prefix" input: "checkpoint_initializer_84/tensor_names" input: "checkpoint_initializer_84/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_84" op: "Assign" input: "bert/encoder/layer_2/output/dense/kernel" input: "checkpoint_initializer_84" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_85/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_85/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_3/attention/output/LayerNorm/beta" } } } } node { name: "checkpoint_initializer_85/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_85" op: "RestoreV2" input: "checkpoint_initializer_85/prefix" input: "checkpoint_initializer_85/tensor_names" input: "checkpoint_initializer_85/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_85" op: "Assign" input: "bert/encoder/layer_3/attention/output/LayerNorm/beta" input: "checkpoint_initializer_85" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_86/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_86/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_3/attention/output/LayerNorm/gamma" } } } } node { name: "checkpoint_initializer_86/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_86" op: "RestoreV2" input: "checkpoint_initializer_86/prefix" input: "checkpoint_initializer_86/tensor_names" input: "checkpoint_initializer_86/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_86" op: "Assign" input: "bert/encoder/layer_3/attention/output/LayerNorm/gamma" input: "checkpoint_initializer_86" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_87/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_87/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_3/attention/output/dense/bias" } } } } node { name: "checkpoint_initializer_87/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_87" op: "RestoreV2" input: "checkpoint_initializer_87/prefix" input: "checkpoint_initializer_87/tensor_names" input: "checkpoint_initializer_87/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_87" op: "Assign" input: "bert/encoder/layer_3/attention/output/dense/bias" input: "checkpoint_initializer_87" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_88/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_88/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_3/attention/output/dense/kernel" } } } } node { name: "checkpoint_initializer_88/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_88" op: "RestoreV2" input: "checkpoint_initializer_88/prefix" input: "checkpoint_initializer_88/tensor_names" input: "checkpoint_initializer_88/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_88" op: "Assign" input: "bert/encoder/layer_3/attention/output/dense/kernel" input: "checkpoint_initializer_88" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_89/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_89/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_3/attention/self/key/bias" } } } } node { name: "checkpoint_initializer_89/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_89" op: "RestoreV2" input: "checkpoint_initializer_89/prefix" input: "checkpoint_initializer_89/tensor_names" input: "checkpoint_initializer_89/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_89" op: "Assign" input: "bert/encoder/layer_3/attention/self/key/bias" input: "checkpoint_initializer_89" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_90/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_90/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_3/attention/self/key/kernel" } } } } node { name: "checkpoint_initializer_90/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_90" op: "RestoreV2" input: "checkpoint_initializer_90/prefix" input: "checkpoint_initializer_90/tensor_names" input: "checkpoint_initializer_90/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_90" op: "Assign" input: "bert/encoder/layer_3/attention/self/key/kernel" input: "checkpoint_initializer_90" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_91/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_91/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_3/attention/self/query/bias" } } } } node { name: "checkpoint_initializer_91/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_91" op: "RestoreV2" input: "checkpoint_initializer_91/prefix" input: "checkpoint_initializer_91/tensor_names" input: "checkpoint_initializer_91/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_91" op: "Assign" input: "bert/encoder/layer_3/attention/self/query/bias" input: "checkpoint_initializer_91" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_92/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_92/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_3/attention/self/query/kernel" } } } } node { name: "checkpoint_initializer_92/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_92" op: "RestoreV2" input: "checkpoint_initializer_92/prefix" input: "checkpoint_initializer_92/tensor_names" input: "checkpoint_initializer_92/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_92" op: "Assign" input: "bert/encoder/layer_3/attention/self/query/kernel" input: "checkpoint_initializer_92" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_93/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_93/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_3/attention/self/value/bias" } } } } node { name: "checkpoint_initializer_93/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_93" op: "RestoreV2" input: "checkpoint_initializer_93/prefix" input: "checkpoint_initializer_93/tensor_names" input: "checkpoint_initializer_93/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_93" op: "Assign" input: "bert/encoder/layer_3/attention/self/value/bias" input: "checkpoint_initializer_93" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_94/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_94/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_3/attention/self/value/kernel" } } } } node { name: "checkpoint_initializer_94/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_94" op: "RestoreV2" input: "checkpoint_initializer_94/prefix" input: "checkpoint_initializer_94/tensor_names" input: "checkpoint_initializer_94/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_94" op: "Assign" input: "bert/encoder/layer_3/attention/self/value/kernel" input: "checkpoint_initializer_94" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_95/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_95/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_3/intermediate/dense/bias" } } } } node { name: "checkpoint_initializer_95/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_95" op: "RestoreV2" input: "checkpoint_initializer_95/prefix" input: "checkpoint_initializer_95/tensor_names" input: "checkpoint_initializer_95/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_95" op: "Assign" input: "bert/encoder/layer_3/intermediate/dense/bias" input: "checkpoint_initializer_95" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_96/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_96/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_3/intermediate/dense/kernel" } } } } node { name: "checkpoint_initializer_96/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_96" op: "RestoreV2" input: "checkpoint_initializer_96/prefix" input: "checkpoint_initializer_96/tensor_names" input: "checkpoint_initializer_96/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_96" op: "Assign" input: "bert/encoder/layer_3/intermediate/dense/kernel" input: "checkpoint_initializer_96" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_97/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_97/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_3/output/LayerNorm/beta" } } } } node { name: "checkpoint_initializer_97/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_97" op: "RestoreV2" input: "checkpoint_initializer_97/prefix" input: "checkpoint_initializer_97/tensor_names" input: "checkpoint_initializer_97/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_97" op: "Assign" input: "bert/encoder/layer_3/output/LayerNorm/beta" input: "checkpoint_initializer_97" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_98/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_98/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_3/output/LayerNorm/gamma" } } } } node { name: "checkpoint_initializer_98/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_98" op: "RestoreV2" input: "checkpoint_initializer_98/prefix" input: "checkpoint_initializer_98/tensor_names" input: "checkpoint_initializer_98/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_98" op: "Assign" input: "bert/encoder/layer_3/output/LayerNorm/gamma" input: "checkpoint_initializer_98" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_99/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_99/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_3/output/dense/bias" } } } } node { name: "checkpoint_initializer_99/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_99" op: "RestoreV2" input: "checkpoint_initializer_99/prefix" input: "checkpoint_initializer_99/tensor_names" input: "checkpoint_initializer_99/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_99" op: "Assign" input: "bert/encoder/layer_3/output/dense/bias" input: "checkpoint_initializer_99" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_100/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_100/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_3/output/dense/kernel" } } } } node { name: "checkpoint_initializer_100/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_100" op: "RestoreV2" input: "checkpoint_initializer_100/prefix" input: "checkpoint_initializer_100/tensor_names" input: "checkpoint_initializer_100/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_100" op: "Assign" input: "bert/encoder/layer_3/output/dense/kernel" input: "checkpoint_initializer_100" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_101/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_101/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_4/attention/output/LayerNorm/beta" } } } } node { name: "checkpoint_initializer_101/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_101" op: "RestoreV2" input: "checkpoint_initializer_101/prefix" input: "checkpoint_initializer_101/tensor_names" input: "checkpoint_initializer_101/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_101" op: "Assign" input: "bert/encoder/layer_4/attention/output/LayerNorm/beta" input: "checkpoint_initializer_101" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_102/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_102/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_4/attention/output/LayerNorm/gamma" } } } } node { name: "checkpoint_initializer_102/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_102" op: "RestoreV2" input: "checkpoint_initializer_102/prefix" input: "checkpoint_initializer_102/tensor_names" input: "checkpoint_initializer_102/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_102" op: "Assign" input: "bert/encoder/layer_4/attention/output/LayerNorm/gamma" input: "checkpoint_initializer_102" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_103/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_103/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_4/attention/output/dense/bias" } } } } node { name: "checkpoint_initializer_103/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_103" op: "RestoreV2" input: "checkpoint_initializer_103/prefix" input: "checkpoint_initializer_103/tensor_names" input: "checkpoint_initializer_103/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_103" op: "Assign" input: "bert/encoder/layer_4/attention/output/dense/bias" input: "checkpoint_initializer_103" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_104/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_104/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_4/attention/output/dense/kernel" } } } } node { name: "checkpoint_initializer_104/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_104" op: "RestoreV2" input: "checkpoint_initializer_104/prefix" input: "checkpoint_initializer_104/tensor_names" input: "checkpoint_initializer_104/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_104" op: "Assign" input: "bert/encoder/layer_4/attention/output/dense/kernel" input: "checkpoint_initializer_104" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_105/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_105/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_4/attention/self/key/bias" } } } } node { name: "checkpoint_initializer_105/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_105" op: "RestoreV2" input: "checkpoint_initializer_105/prefix" input: "checkpoint_initializer_105/tensor_names" input: "checkpoint_initializer_105/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_105" op: "Assign" input: "bert/encoder/layer_4/attention/self/key/bias" input: "checkpoint_initializer_105" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_106/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_106/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_4/attention/self/key/kernel" } } } } node { name: "checkpoint_initializer_106/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_106" op: "RestoreV2" input: "checkpoint_initializer_106/prefix" input: "checkpoint_initializer_106/tensor_names" input: "checkpoint_initializer_106/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_106" op: "Assign" input: "bert/encoder/layer_4/attention/self/key/kernel" input: "checkpoint_initializer_106" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_107/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_107/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_4/attention/self/query/bias" } } } } node { name: "checkpoint_initializer_107/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_107" op: "RestoreV2" input: "checkpoint_initializer_107/prefix" input: "checkpoint_initializer_107/tensor_names" input: "checkpoint_initializer_107/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_107" op: "Assign" input: "bert/encoder/layer_4/attention/self/query/bias" input: "checkpoint_initializer_107" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_108/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_108/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_4/attention/self/query/kernel" } } } } node { name: "checkpoint_initializer_108/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_108" op: "RestoreV2" input: "checkpoint_initializer_108/prefix" input: "checkpoint_initializer_108/tensor_names" input: "checkpoint_initializer_108/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_108" op: "Assign" input: "bert/encoder/layer_4/attention/self/query/kernel" input: "checkpoint_initializer_108" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_109/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_109/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_4/attention/self/value/bias" } } } } node { name: "checkpoint_initializer_109/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_109" op: "RestoreV2" input: "checkpoint_initializer_109/prefix" input: "checkpoint_initializer_109/tensor_names" input: "checkpoint_initializer_109/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_109" op: "Assign" input: "bert/encoder/layer_4/attention/self/value/bias" input: "checkpoint_initializer_109" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_110/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_110/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_4/attention/self/value/kernel" } } } } node { name: "checkpoint_initializer_110/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_110" op: "RestoreV2" input: "checkpoint_initializer_110/prefix" input: "checkpoint_initializer_110/tensor_names" input: "checkpoint_initializer_110/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_110" op: "Assign" input: "bert/encoder/layer_4/attention/self/value/kernel" input: "checkpoint_initializer_110" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_111/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_111/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_4/intermediate/dense/bias" } } } } node { name: "checkpoint_initializer_111/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_111" op: "RestoreV2" input: "checkpoint_initializer_111/prefix" input: "checkpoint_initializer_111/tensor_names" input: "checkpoint_initializer_111/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_111" op: "Assign" input: "bert/encoder/layer_4/intermediate/dense/bias" input: "checkpoint_initializer_111" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_112/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_112/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_4/intermediate/dense/kernel" } } } } node { name: "checkpoint_initializer_112/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_112" op: "RestoreV2" input: "checkpoint_initializer_112/prefix" input: "checkpoint_initializer_112/tensor_names" input: "checkpoint_initializer_112/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_112" op: "Assign" input: "bert/encoder/layer_4/intermediate/dense/kernel" input: "checkpoint_initializer_112" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_113/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_113/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_4/output/LayerNorm/beta" } } } } node { name: "checkpoint_initializer_113/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_113" op: "RestoreV2" input: "checkpoint_initializer_113/prefix" input: "checkpoint_initializer_113/tensor_names" input: "checkpoint_initializer_113/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_113" op: "Assign" input: "bert/encoder/layer_4/output/LayerNorm/beta" input: "checkpoint_initializer_113" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_114/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_114/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_4/output/LayerNorm/gamma" } } } } node { name: "checkpoint_initializer_114/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_114" op: "RestoreV2" input: "checkpoint_initializer_114/prefix" input: "checkpoint_initializer_114/tensor_names" input: "checkpoint_initializer_114/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_114" op: "Assign" input: "bert/encoder/layer_4/output/LayerNorm/gamma" input: "checkpoint_initializer_114" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_115/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_115/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_4/output/dense/bias" } } } } node { name: "checkpoint_initializer_115/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_115" op: "RestoreV2" input: "checkpoint_initializer_115/prefix" input: "checkpoint_initializer_115/tensor_names" input: "checkpoint_initializer_115/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_115" op: "Assign" input: "bert/encoder/layer_4/output/dense/bias" input: "checkpoint_initializer_115" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_116/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_116/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_4/output/dense/kernel" } } } } node { name: "checkpoint_initializer_116/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_116" op: "RestoreV2" input: "checkpoint_initializer_116/prefix" input: "checkpoint_initializer_116/tensor_names" input: "checkpoint_initializer_116/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_116" op: "Assign" input: "bert/encoder/layer_4/output/dense/kernel" input: "checkpoint_initializer_116" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_117/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_117/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_5/attention/output/LayerNorm/beta" } } } } node { name: "checkpoint_initializer_117/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_117" op: "RestoreV2" input: "checkpoint_initializer_117/prefix" input: "checkpoint_initializer_117/tensor_names" input: "checkpoint_initializer_117/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_117" op: "Assign" input: "bert/encoder/layer_5/attention/output/LayerNorm/beta" input: "checkpoint_initializer_117" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_118/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_118/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_5/attention/output/LayerNorm/gamma" } } } } node { name: "checkpoint_initializer_118/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_118" op: "RestoreV2" input: "checkpoint_initializer_118/prefix" input: "checkpoint_initializer_118/tensor_names" input: "checkpoint_initializer_118/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_118" op: "Assign" input: "bert/encoder/layer_5/attention/output/LayerNorm/gamma" input: "checkpoint_initializer_118" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_119/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_119/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_5/attention/output/dense/bias" } } } } node { name: "checkpoint_initializer_119/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_119" op: "RestoreV2" input: "checkpoint_initializer_119/prefix" input: "checkpoint_initializer_119/tensor_names" input: "checkpoint_initializer_119/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_119" op: "Assign" input: "bert/encoder/layer_5/attention/output/dense/bias" input: "checkpoint_initializer_119" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_120/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_120/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_5/attention/output/dense/kernel" } } } } node { name: "checkpoint_initializer_120/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_120" op: "RestoreV2" input: "checkpoint_initializer_120/prefix" input: "checkpoint_initializer_120/tensor_names" input: "checkpoint_initializer_120/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_120" op: "Assign" input: "bert/encoder/layer_5/attention/output/dense/kernel" input: "checkpoint_initializer_120" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_121/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_121/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_5/attention/self/key/bias" } } } } node { name: "checkpoint_initializer_121/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_121" op: "RestoreV2" input: "checkpoint_initializer_121/prefix" input: "checkpoint_initializer_121/tensor_names" input: "checkpoint_initializer_121/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_121" op: "Assign" input: "bert/encoder/layer_5/attention/self/key/bias" input: "checkpoint_initializer_121" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_122/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_122/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_5/attention/self/key/kernel" } } } } node { name: "checkpoint_initializer_122/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_122" op: "RestoreV2" input: "checkpoint_initializer_122/prefix" input: "checkpoint_initializer_122/tensor_names" input: "checkpoint_initializer_122/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_122" op: "Assign" input: "bert/encoder/layer_5/attention/self/key/kernel" input: "checkpoint_initializer_122" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_123/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_123/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_5/attention/self/query/bias" } } } } node { name: "checkpoint_initializer_123/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_123" op: "RestoreV2" input: "checkpoint_initializer_123/prefix" input: "checkpoint_initializer_123/tensor_names" input: "checkpoint_initializer_123/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_123" op: "Assign" input: "bert/encoder/layer_5/attention/self/query/bias" input: "checkpoint_initializer_123" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_124/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_124/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_5/attention/self/query/kernel" } } } } node { name: "checkpoint_initializer_124/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_124" op: "RestoreV2" input: "checkpoint_initializer_124/prefix" input: "checkpoint_initializer_124/tensor_names" input: "checkpoint_initializer_124/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_124" op: "Assign" input: "bert/encoder/layer_5/attention/self/query/kernel" input: "checkpoint_initializer_124" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_125/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_125/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_5/attention/self/value/bias" } } } } node { name: "checkpoint_initializer_125/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_125" op: "RestoreV2" input: "checkpoint_initializer_125/prefix" input: "checkpoint_initializer_125/tensor_names" input: "checkpoint_initializer_125/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_125" op: "Assign" input: "bert/encoder/layer_5/attention/self/value/bias" input: "checkpoint_initializer_125" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_126/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_126/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_5/attention/self/value/kernel" } } } } node { name: "checkpoint_initializer_126/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_126" op: "RestoreV2" input: "checkpoint_initializer_126/prefix" input: "checkpoint_initializer_126/tensor_names" input: "checkpoint_initializer_126/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_126" op: "Assign" input: "bert/encoder/layer_5/attention/self/value/kernel" input: "checkpoint_initializer_126" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_127/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_127/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_5/intermediate/dense/bias" } } } } node { name: "checkpoint_initializer_127/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_127" op: "RestoreV2" input: "checkpoint_initializer_127/prefix" input: "checkpoint_initializer_127/tensor_names" input: "checkpoint_initializer_127/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_127" op: "Assign" input: "bert/encoder/layer_5/intermediate/dense/bias" input: "checkpoint_initializer_127" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_128/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_128/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_5/intermediate/dense/kernel" } } } } node { name: "checkpoint_initializer_128/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_128" op: "RestoreV2" input: "checkpoint_initializer_128/prefix" input: "checkpoint_initializer_128/tensor_names" input: "checkpoint_initializer_128/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_128" op: "Assign" input: "bert/encoder/layer_5/intermediate/dense/kernel" input: "checkpoint_initializer_128" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_129/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_129/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_5/output/LayerNorm/beta" } } } } node { name: "checkpoint_initializer_129/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_129" op: "RestoreV2" input: "checkpoint_initializer_129/prefix" input: "checkpoint_initializer_129/tensor_names" input: "checkpoint_initializer_129/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_129" op: "Assign" input: "bert/encoder/layer_5/output/LayerNorm/beta" input: "checkpoint_initializer_129" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_130/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_130/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_5/output/LayerNorm/gamma" } } } } node { name: "checkpoint_initializer_130/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_130" op: "RestoreV2" input: "checkpoint_initializer_130/prefix" input: "checkpoint_initializer_130/tensor_names" input: "checkpoint_initializer_130/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_130" op: "Assign" input: "bert/encoder/layer_5/output/LayerNorm/gamma" input: "checkpoint_initializer_130" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_131/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_131/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_5/output/dense/bias" } } } } node { name: "checkpoint_initializer_131/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_131" op: "RestoreV2" input: "checkpoint_initializer_131/prefix" input: "checkpoint_initializer_131/tensor_names" input: "checkpoint_initializer_131/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_131" op: "Assign" input: "bert/encoder/layer_5/output/dense/bias" input: "checkpoint_initializer_131" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_132/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_132/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_5/output/dense/kernel" } } } } node { name: "checkpoint_initializer_132/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_132" op: "RestoreV2" input: "checkpoint_initializer_132/prefix" input: "checkpoint_initializer_132/tensor_names" input: "checkpoint_initializer_132/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_132" op: "Assign" input: "bert/encoder/layer_5/output/dense/kernel" input: "checkpoint_initializer_132" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_133/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_133/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_6/attention/output/LayerNorm/beta" } } } } node { name: "checkpoint_initializer_133/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_133" op: "RestoreV2" input: "checkpoint_initializer_133/prefix" input: "checkpoint_initializer_133/tensor_names" input: "checkpoint_initializer_133/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_133" op: "Assign" input: "bert/encoder/layer_6/attention/output/LayerNorm/beta" input: "checkpoint_initializer_133" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_134/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_134/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_6/attention/output/LayerNorm/gamma" } } } } node { name: "checkpoint_initializer_134/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_134" op: "RestoreV2" input: "checkpoint_initializer_134/prefix" input: "checkpoint_initializer_134/tensor_names" input: "checkpoint_initializer_134/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_134" op: "Assign" input: "bert/encoder/layer_6/attention/output/LayerNorm/gamma" input: "checkpoint_initializer_134" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_135/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_135/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_6/attention/output/dense/bias" } } } } node { name: "checkpoint_initializer_135/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_135" op: "RestoreV2" input: "checkpoint_initializer_135/prefix" input: "checkpoint_initializer_135/tensor_names" input: "checkpoint_initializer_135/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_135" op: "Assign" input: "bert/encoder/layer_6/attention/output/dense/bias" input: "checkpoint_initializer_135" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_136/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_136/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_6/attention/output/dense/kernel" } } } } node { name: "checkpoint_initializer_136/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_136" op: "RestoreV2" input: "checkpoint_initializer_136/prefix" input: "checkpoint_initializer_136/tensor_names" input: "checkpoint_initializer_136/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_136" op: "Assign" input: "bert/encoder/layer_6/attention/output/dense/kernel" input: "checkpoint_initializer_136" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_137/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_137/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_6/attention/self/key/bias" } } } } node { name: "checkpoint_initializer_137/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_137" op: "RestoreV2" input: "checkpoint_initializer_137/prefix" input: "checkpoint_initializer_137/tensor_names" input: "checkpoint_initializer_137/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_137" op: "Assign" input: "bert/encoder/layer_6/attention/self/key/bias" input: "checkpoint_initializer_137" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_138/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_138/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_6/attention/self/key/kernel" } } } } node { name: "checkpoint_initializer_138/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_138" op: "RestoreV2" input: "checkpoint_initializer_138/prefix" input: "checkpoint_initializer_138/tensor_names" input: "checkpoint_initializer_138/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_138" op: "Assign" input: "bert/encoder/layer_6/attention/self/key/kernel" input: "checkpoint_initializer_138" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_139/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_139/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_6/attention/self/query/bias" } } } } node { name: "checkpoint_initializer_139/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_139" op: "RestoreV2" input: "checkpoint_initializer_139/prefix" input: "checkpoint_initializer_139/tensor_names" input: "checkpoint_initializer_139/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_139" op: "Assign" input: "bert/encoder/layer_6/attention/self/query/bias" input: "checkpoint_initializer_139" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_140/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_140/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_6/attention/self/query/kernel" } } } } node { name: "checkpoint_initializer_140/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_140" op: "RestoreV2" input: "checkpoint_initializer_140/prefix" input: "checkpoint_initializer_140/tensor_names" input: "checkpoint_initializer_140/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_140" op: "Assign" input: "bert/encoder/layer_6/attention/self/query/kernel" input: "checkpoint_initializer_140" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_141/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_141/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_6/attention/self/value/bias" } } } } node { name: "checkpoint_initializer_141/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_141" op: "RestoreV2" input: "checkpoint_initializer_141/prefix" input: "checkpoint_initializer_141/tensor_names" input: "checkpoint_initializer_141/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_141" op: "Assign" input: "bert/encoder/layer_6/attention/self/value/bias" input: "checkpoint_initializer_141" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_142/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_142/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_6/attention/self/value/kernel" } } } } node { name: "checkpoint_initializer_142/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_142" op: "RestoreV2" input: "checkpoint_initializer_142/prefix" input: "checkpoint_initializer_142/tensor_names" input: "checkpoint_initializer_142/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_142" op: "Assign" input: "bert/encoder/layer_6/attention/self/value/kernel" input: "checkpoint_initializer_142" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_143/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_143/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_6/intermediate/dense/bias" } } } } node { name: "checkpoint_initializer_143/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_143" op: "RestoreV2" input: "checkpoint_initializer_143/prefix" input: "checkpoint_initializer_143/tensor_names" input: "checkpoint_initializer_143/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_143" op: "Assign" input: "bert/encoder/layer_6/intermediate/dense/bias" input: "checkpoint_initializer_143" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_144/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_144/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_6/intermediate/dense/kernel" } } } } node { name: "checkpoint_initializer_144/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_144" op: "RestoreV2" input: "checkpoint_initializer_144/prefix" input: "checkpoint_initializer_144/tensor_names" input: "checkpoint_initializer_144/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_144" op: "Assign" input: "bert/encoder/layer_6/intermediate/dense/kernel" input: "checkpoint_initializer_144" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_145/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_145/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_6/output/LayerNorm/beta" } } } } node { name: "checkpoint_initializer_145/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_145" op: "RestoreV2" input: "checkpoint_initializer_145/prefix" input: "checkpoint_initializer_145/tensor_names" input: "checkpoint_initializer_145/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_145" op: "Assign" input: "bert/encoder/layer_6/output/LayerNorm/beta" input: "checkpoint_initializer_145" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_146/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_146/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_6/output/LayerNorm/gamma" } } } } node { name: "checkpoint_initializer_146/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_146" op: "RestoreV2" input: "checkpoint_initializer_146/prefix" input: "checkpoint_initializer_146/tensor_names" input: "checkpoint_initializer_146/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_146" op: "Assign" input: "bert/encoder/layer_6/output/LayerNorm/gamma" input: "checkpoint_initializer_146" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_147/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_147/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_6/output/dense/bias" } } } } node { name: "checkpoint_initializer_147/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_147" op: "RestoreV2" input: "checkpoint_initializer_147/prefix" input: "checkpoint_initializer_147/tensor_names" input: "checkpoint_initializer_147/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_147" op: "Assign" input: "bert/encoder/layer_6/output/dense/bias" input: "checkpoint_initializer_147" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_148/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_148/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_6/output/dense/kernel" } } } } node { name: "checkpoint_initializer_148/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_148" op: "RestoreV2" input: "checkpoint_initializer_148/prefix" input: "checkpoint_initializer_148/tensor_names" input: "checkpoint_initializer_148/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_148" op: "Assign" input: "bert/encoder/layer_6/output/dense/kernel" input: "checkpoint_initializer_148" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_149/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_149/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_7/attention/output/LayerNorm/beta" } } } } node { name: "checkpoint_initializer_149/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_149" op: "RestoreV2" input: "checkpoint_initializer_149/prefix" input: "checkpoint_initializer_149/tensor_names" input: "checkpoint_initializer_149/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_149" op: "Assign" input: "bert/encoder/layer_7/attention/output/LayerNorm/beta" input: "checkpoint_initializer_149" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_150/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_150/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_7/attention/output/LayerNorm/gamma" } } } } node { name: "checkpoint_initializer_150/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_150" op: "RestoreV2" input: "checkpoint_initializer_150/prefix" input: "checkpoint_initializer_150/tensor_names" input: "checkpoint_initializer_150/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_150" op: "Assign" input: "bert/encoder/layer_7/attention/output/LayerNorm/gamma" input: "checkpoint_initializer_150" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_151/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_151/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_7/attention/output/dense/bias" } } } } node { name: "checkpoint_initializer_151/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_151" op: "RestoreV2" input: "checkpoint_initializer_151/prefix" input: "checkpoint_initializer_151/tensor_names" input: "checkpoint_initializer_151/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_151" op: "Assign" input: "bert/encoder/layer_7/attention/output/dense/bias" input: "checkpoint_initializer_151" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_152/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_152/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_7/attention/output/dense/kernel" } } } } node { name: "checkpoint_initializer_152/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_152" op: "RestoreV2" input: "checkpoint_initializer_152/prefix" input: "checkpoint_initializer_152/tensor_names" input: "checkpoint_initializer_152/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_152" op: "Assign" input: "bert/encoder/layer_7/attention/output/dense/kernel" input: "checkpoint_initializer_152" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_153/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_153/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_7/attention/self/key/bias" } } } } node { name: "checkpoint_initializer_153/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_153" op: "RestoreV2" input: "checkpoint_initializer_153/prefix" input: "checkpoint_initializer_153/tensor_names" input: "checkpoint_initializer_153/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_153" op: "Assign" input: "bert/encoder/layer_7/attention/self/key/bias" input: "checkpoint_initializer_153" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_154/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_154/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_7/attention/self/key/kernel" } } } } node { name: "checkpoint_initializer_154/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_154" op: "RestoreV2" input: "checkpoint_initializer_154/prefix" input: "checkpoint_initializer_154/tensor_names" input: "checkpoint_initializer_154/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_154" op: "Assign" input: "bert/encoder/layer_7/attention/self/key/kernel" input: "checkpoint_initializer_154" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_155/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_155/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_7/attention/self/query/bias" } } } } node { name: "checkpoint_initializer_155/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_155" op: "RestoreV2" input: "checkpoint_initializer_155/prefix" input: "checkpoint_initializer_155/tensor_names" input: "checkpoint_initializer_155/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_155" op: "Assign" input: "bert/encoder/layer_7/attention/self/query/bias" input: "checkpoint_initializer_155" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_156/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_156/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_7/attention/self/query/kernel" } } } } node { name: "checkpoint_initializer_156/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_156" op: "RestoreV2" input: "checkpoint_initializer_156/prefix" input: "checkpoint_initializer_156/tensor_names" input: "checkpoint_initializer_156/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_156" op: "Assign" input: "bert/encoder/layer_7/attention/self/query/kernel" input: "checkpoint_initializer_156" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_157/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_157/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_7/attention/self/value/bias" } } } } node { name: "checkpoint_initializer_157/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_157" op: "RestoreV2" input: "checkpoint_initializer_157/prefix" input: "checkpoint_initializer_157/tensor_names" input: "checkpoint_initializer_157/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_157" op: "Assign" input: "bert/encoder/layer_7/attention/self/value/bias" input: "checkpoint_initializer_157" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_158/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_158/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_7/attention/self/value/kernel" } } } } node { name: "checkpoint_initializer_158/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_158" op: "RestoreV2" input: "checkpoint_initializer_158/prefix" input: "checkpoint_initializer_158/tensor_names" input: "checkpoint_initializer_158/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_158" op: "Assign" input: "bert/encoder/layer_7/attention/self/value/kernel" input: "checkpoint_initializer_158" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_159/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_159/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_7/intermediate/dense/bias" } } } } node { name: "checkpoint_initializer_159/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_159" op: "RestoreV2" input: "checkpoint_initializer_159/prefix" input: "checkpoint_initializer_159/tensor_names" input: "checkpoint_initializer_159/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_159" op: "Assign" input: "bert/encoder/layer_7/intermediate/dense/bias" input: "checkpoint_initializer_159" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_160/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_160/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_7/intermediate/dense/kernel" } } } } node { name: "checkpoint_initializer_160/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_160" op: "RestoreV2" input: "checkpoint_initializer_160/prefix" input: "checkpoint_initializer_160/tensor_names" input: "checkpoint_initializer_160/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_160" op: "Assign" input: "bert/encoder/layer_7/intermediate/dense/kernel" input: "checkpoint_initializer_160" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_161/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_161/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_7/output/LayerNorm/beta" } } } } node { name: "checkpoint_initializer_161/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_161" op: "RestoreV2" input: "checkpoint_initializer_161/prefix" input: "checkpoint_initializer_161/tensor_names" input: "checkpoint_initializer_161/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_161" op: "Assign" input: "bert/encoder/layer_7/output/LayerNorm/beta" input: "checkpoint_initializer_161" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_162/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_162/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_7/output/LayerNorm/gamma" } } } } node { name: "checkpoint_initializer_162/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_162" op: "RestoreV2" input: "checkpoint_initializer_162/prefix" input: "checkpoint_initializer_162/tensor_names" input: "checkpoint_initializer_162/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_162" op: "Assign" input: "bert/encoder/layer_7/output/LayerNorm/gamma" input: "checkpoint_initializer_162" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_163/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_163/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_7/output/dense/bias" } } } } node { name: "checkpoint_initializer_163/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_163" op: "RestoreV2" input: "checkpoint_initializer_163/prefix" input: "checkpoint_initializer_163/tensor_names" input: "checkpoint_initializer_163/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_163" op: "Assign" input: "bert/encoder/layer_7/output/dense/bias" input: "checkpoint_initializer_163" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_164/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_164/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_7/output/dense/kernel" } } } } node { name: "checkpoint_initializer_164/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_164" op: "RestoreV2" input: "checkpoint_initializer_164/prefix" input: "checkpoint_initializer_164/tensor_names" input: "checkpoint_initializer_164/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_164" op: "Assign" input: "bert/encoder/layer_7/output/dense/kernel" input: "checkpoint_initializer_164" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_165/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_165/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_8/attention/output/LayerNorm/beta" } } } } node { name: "checkpoint_initializer_165/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_165" op: "RestoreV2" input: "checkpoint_initializer_165/prefix" input: "checkpoint_initializer_165/tensor_names" input: "checkpoint_initializer_165/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_165" op: "Assign" input: "bert/encoder/layer_8/attention/output/LayerNorm/beta" input: "checkpoint_initializer_165" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_166/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_166/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_8/attention/output/LayerNorm/gamma" } } } } node { name: "checkpoint_initializer_166/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_166" op: "RestoreV2" input: "checkpoint_initializer_166/prefix" input: "checkpoint_initializer_166/tensor_names" input: "checkpoint_initializer_166/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_166" op: "Assign" input: "bert/encoder/layer_8/attention/output/LayerNorm/gamma" input: "checkpoint_initializer_166" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_167/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_167/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_8/attention/output/dense/bias" } } } } node { name: "checkpoint_initializer_167/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_167" op: "RestoreV2" input: "checkpoint_initializer_167/prefix" input: "checkpoint_initializer_167/tensor_names" input: "checkpoint_initializer_167/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_167" op: "Assign" input: "bert/encoder/layer_8/attention/output/dense/bias" input: "checkpoint_initializer_167" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_168/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_168/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_8/attention/output/dense/kernel" } } } } node { name: "checkpoint_initializer_168/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_168" op: "RestoreV2" input: "checkpoint_initializer_168/prefix" input: "checkpoint_initializer_168/tensor_names" input: "checkpoint_initializer_168/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_168" op: "Assign" input: "bert/encoder/layer_8/attention/output/dense/kernel" input: "checkpoint_initializer_168" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_169/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_169/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_8/attention/self/key/bias" } } } } node { name: "checkpoint_initializer_169/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_169" op: "RestoreV2" input: "checkpoint_initializer_169/prefix" input: "checkpoint_initializer_169/tensor_names" input: "checkpoint_initializer_169/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_169" op: "Assign" input: "bert/encoder/layer_8/attention/self/key/bias" input: "checkpoint_initializer_169" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_170/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_170/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_8/attention/self/key/kernel" } } } } node { name: "checkpoint_initializer_170/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_170" op: "RestoreV2" input: "checkpoint_initializer_170/prefix" input: "checkpoint_initializer_170/tensor_names" input: "checkpoint_initializer_170/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_170" op: "Assign" input: "bert/encoder/layer_8/attention/self/key/kernel" input: "checkpoint_initializer_170" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_171/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_171/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_8/attention/self/query/bias" } } } } node { name: "checkpoint_initializer_171/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_171" op: "RestoreV2" input: "checkpoint_initializer_171/prefix" input: "checkpoint_initializer_171/tensor_names" input: "checkpoint_initializer_171/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_171" op: "Assign" input: "bert/encoder/layer_8/attention/self/query/bias" input: "checkpoint_initializer_171" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_172/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_172/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_8/attention/self/query/kernel" } } } } node { name: "checkpoint_initializer_172/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_172" op: "RestoreV2" input: "checkpoint_initializer_172/prefix" input: "checkpoint_initializer_172/tensor_names" input: "checkpoint_initializer_172/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_172" op: "Assign" input: "bert/encoder/layer_8/attention/self/query/kernel" input: "checkpoint_initializer_172" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_173/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_173/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_8/attention/self/value/bias" } } } } node { name: "checkpoint_initializer_173/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_173" op: "RestoreV2" input: "checkpoint_initializer_173/prefix" input: "checkpoint_initializer_173/tensor_names" input: "checkpoint_initializer_173/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_173" op: "Assign" input: "bert/encoder/layer_8/attention/self/value/bias" input: "checkpoint_initializer_173" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_174/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_174/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_8/attention/self/value/kernel" } } } } node { name: "checkpoint_initializer_174/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_174" op: "RestoreV2" input: "checkpoint_initializer_174/prefix" input: "checkpoint_initializer_174/tensor_names" input: "checkpoint_initializer_174/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_174" op: "Assign" input: "bert/encoder/layer_8/attention/self/value/kernel" input: "checkpoint_initializer_174" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_175/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_175/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_8/intermediate/dense/bias" } } } } node { name: "checkpoint_initializer_175/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_175" op: "RestoreV2" input: "checkpoint_initializer_175/prefix" input: "checkpoint_initializer_175/tensor_names" input: "checkpoint_initializer_175/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_175" op: "Assign" input: "bert/encoder/layer_8/intermediate/dense/bias" input: "checkpoint_initializer_175" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_176/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_176/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_8/intermediate/dense/kernel" } } } } node { name: "checkpoint_initializer_176/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_176" op: "RestoreV2" input: "checkpoint_initializer_176/prefix" input: "checkpoint_initializer_176/tensor_names" input: "checkpoint_initializer_176/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_176" op: "Assign" input: "bert/encoder/layer_8/intermediate/dense/kernel" input: "checkpoint_initializer_176" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_177/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_177/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_8/output/LayerNorm/beta" } } } } node { name: "checkpoint_initializer_177/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_177" op: "RestoreV2" input: "checkpoint_initializer_177/prefix" input: "checkpoint_initializer_177/tensor_names" input: "checkpoint_initializer_177/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_177" op: "Assign" input: "bert/encoder/layer_8/output/LayerNorm/beta" input: "checkpoint_initializer_177" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_178/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_178/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_8/output/LayerNorm/gamma" } } } } node { name: "checkpoint_initializer_178/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_178" op: "RestoreV2" input: "checkpoint_initializer_178/prefix" input: "checkpoint_initializer_178/tensor_names" input: "checkpoint_initializer_178/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_178" op: "Assign" input: "bert/encoder/layer_8/output/LayerNorm/gamma" input: "checkpoint_initializer_178" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_179/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_179/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_8/output/dense/bias" } } } } node { name: "checkpoint_initializer_179/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_179" op: "RestoreV2" input: "checkpoint_initializer_179/prefix" input: "checkpoint_initializer_179/tensor_names" input: "checkpoint_initializer_179/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_179" op: "Assign" input: "bert/encoder/layer_8/output/dense/bias" input: "checkpoint_initializer_179" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_180/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_180/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_8/output/dense/kernel" } } } } node { name: "checkpoint_initializer_180/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_180" op: "RestoreV2" input: "checkpoint_initializer_180/prefix" input: "checkpoint_initializer_180/tensor_names" input: "checkpoint_initializer_180/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_180" op: "Assign" input: "bert/encoder/layer_8/output/dense/kernel" input: "checkpoint_initializer_180" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_181/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_181/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_9/attention/output/LayerNorm/beta" } } } } node { name: "checkpoint_initializer_181/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_181" op: "RestoreV2" input: "checkpoint_initializer_181/prefix" input: "checkpoint_initializer_181/tensor_names" input: "checkpoint_initializer_181/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_181" op: "Assign" input: "bert/encoder/layer_9/attention/output/LayerNorm/beta" input: "checkpoint_initializer_181" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_182/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_182/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_9/attention/output/LayerNorm/gamma" } } } } node { name: "checkpoint_initializer_182/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_182" op: "RestoreV2" input: "checkpoint_initializer_182/prefix" input: "checkpoint_initializer_182/tensor_names" input: "checkpoint_initializer_182/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_182" op: "Assign" input: "bert/encoder/layer_9/attention/output/LayerNorm/gamma" input: "checkpoint_initializer_182" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_183/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_183/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_9/attention/output/dense/bias" } } } } node { name: "checkpoint_initializer_183/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_183" op: "RestoreV2" input: "checkpoint_initializer_183/prefix" input: "checkpoint_initializer_183/tensor_names" input: "checkpoint_initializer_183/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_183" op: "Assign" input: "bert/encoder/layer_9/attention/output/dense/bias" input: "checkpoint_initializer_183" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_184/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_184/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_9/attention/output/dense/kernel" } } } } node { name: "checkpoint_initializer_184/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_184" op: "RestoreV2" input: "checkpoint_initializer_184/prefix" input: "checkpoint_initializer_184/tensor_names" input: "checkpoint_initializer_184/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_184" op: "Assign" input: "bert/encoder/layer_9/attention/output/dense/kernel" input: "checkpoint_initializer_184" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_185/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_185/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_9/attention/self/key/bias" } } } } node { name: "checkpoint_initializer_185/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_185" op: "RestoreV2" input: "checkpoint_initializer_185/prefix" input: "checkpoint_initializer_185/tensor_names" input: "checkpoint_initializer_185/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_185" op: "Assign" input: "bert/encoder/layer_9/attention/self/key/bias" input: "checkpoint_initializer_185" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_186/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_186/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_9/attention/self/key/kernel" } } } } node { name: "checkpoint_initializer_186/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_186" op: "RestoreV2" input: "checkpoint_initializer_186/prefix" input: "checkpoint_initializer_186/tensor_names" input: "checkpoint_initializer_186/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_186" op: "Assign" input: "bert/encoder/layer_9/attention/self/key/kernel" input: "checkpoint_initializer_186" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_187/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_187/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_9/attention/self/query/bias" } } } } node { name: "checkpoint_initializer_187/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_187" op: "RestoreV2" input: "checkpoint_initializer_187/prefix" input: "checkpoint_initializer_187/tensor_names" input: "checkpoint_initializer_187/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_187" op: "Assign" input: "bert/encoder/layer_9/attention/self/query/bias" input: "checkpoint_initializer_187" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_188/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_188/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_9/attention/self/query/kernel" } } } } node { name: "checkpoint_initializer_188/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_188" op: "RestoreV2" input: "checkpoint_initializer_188/prefix" input: "checkpoint_initializer_188/tensor_names" input: "checkpoint_initializer_188/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_188" op: "Assign" input: "bert/encoder/layer_9/attention/self/query/kernel" input: "checkpoint_initializer_188" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_189/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_189/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_9/attention/self/value/bias" } } } } node { name: "checkpoint_initializer_189/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_189" op: "RestoreV2" input: "checkpoint_initializer_189/prefix" input: "checkpoint_initializer_189/tensor_names" input: "checkpoint_initializer_189/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_189" op: "Assign" input: "bert/encoder/layer_9/attention/self/value/bias" input: "checkpoint_initializer_189" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_190/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_190/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_9/attention/self/value/kernel" } } } } node { name: "checkpoint_initializer_190/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_190" op: "RestoreV2" input: "checkpoint_initializer_190/prefix" input: "checkpoint_initializer_190/tensor_names" input: "checkpoint_initializer_190/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_190" op: "Assign" input: "bert/encoder/layer_9/attention/self/value/kernel" input: "checkpoint_initializer_190" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_191/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_191/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_9/intermediate/dense/bias" } } } } node { name: "checkpoint_initializer_191/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_191" op: "RestoreV2" input: "checkpoint_initializer_191/prefix" input: "checkpoint_initializer_191/tensor_names" input: "checkpoint_initializer_191/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_191" op: "Assign" input: "bert/encoder/layer_9/intermediate/dense/bias" input: "checkpoint_initializer_191" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_192/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_192/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_9/intermediate/dense/kernel" } } } } node { name: "checkpoint_initializer_192/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_192" op: "RestoreV2" input: "checkpoint_initializer_192/prefix" input: "checkpoint_initializer_192/tensor_names" input: "checkpoint_initializer_192/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_192" op: "Assign" input: "bert/encoder/layer_9/intermediate/dense/kernel" input: "checkpoint_initializer_192" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_193/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_193/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_9/output/LayerNorm/beta" } } } } node { name: "checkpoint_initializer_193/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_193" op: "RestoreV2" input: "checkpoint_initializer_193/prefix" input: "checkpoint_initializer_193/tensor_names" input: "checkpoint_initializer_193/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_193" op: "Assign" input: "bert/encoder/layer_9/output/LayerNorm/beta" input: "checkpoint_initializer_193" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_194/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_194/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_9/output/LayerNorm/gamma" } } } } node { name: "checkpoint_initializer_194/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_194" op: "RestoreV2" input: "checkpoint_initializer_194/prefix" input: "checkpoint_initializer_194/tensor_names" input: "checkpoint_initializer_194/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_194" op: "Assign" input: "bert/encoder/layer_9/output/LayerNorm/gamma" input: "checkpoint_initializer_194" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_195/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_195/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_9/output/dense/bias" } } } } node { name: "checkpoint_initializer_195/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_195" op: "RestoreV2" input: "checkpoint_initializer_195/prefix" input: "checkpoint_initializer_195/tensor_names" input: "checkpoint_initializer_195/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_195" op: "Assign" input: "bert/encoder/layer_9/output/dense/bias" input: "checkpoint_initializer_195" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_196/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_196/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_9/output/dense/kernel" } } } } node { name: "checkpoint_initializer_196/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_196" op: "RestoreV2" input: "checkpoint_initializer_196/prefix" input: "checkpoint_initializer_196/tensor_names" input: "checkpoint_initializer_196/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_196" op: "Assign" input: "bert/encoder/layer_9/output/dense/kernel" input: "checkpoint_initializer_196" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_197/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_197/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/pooler/dense/bias" } } } } node { name: "checkpoint_initializer_197/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_197" op: "RestoreV2" input: "checkpoint_initializer_197/prefix" input: "checkpoint_initializer_197/tensor_names" input: "checkpoint_initializer_197/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_197" op: "Assign" input: "bert/pooler/dense/bias" input: "checkpoint_initializer_197" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_198/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_198/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/pooler/dense/kernel" } } } } node { name: "checkpoint_initializer_198/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_198" op: "RestoreV2" input: "checkpoint_initializer_198/prefix" input: "checkpoint_initializer_198/tensor_names" input: "checkpoint_initializer_198/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_198" op: "Assign" input: "bert/pooler/dense/kernel" input: "checkpoint_initializer_198" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_199/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_199/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "cls/predictions/output_bias" } } } } node { name: "checkpoint_initializer_199/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_199" op: "RestoreV2" input: "checkpoint_initializer_199/prefix" input: "checkpoint_initializer_199/tensor_names" input: "checkpoint_initializer_199/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_199" op: "Assign" input: "cls/predictions/output_bias" input: "checkpoint_initializer_199" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/output_bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_200/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_200/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "cls/predictions/transform/LayerNorm/beta" } } } } node { name: "checkpoint_initializer_200/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_200" op: "RestoreV2" input: "checkpoint_initializer_200/prefix" input: "checkpoint_initializer_200/tensor_names" input: "checkpoint_initializer_200/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_200" op: "Assign" input: "cls/predictions/transform/LayerNorm/beta" input: "checkpoint_initializer_200" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_201/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_201/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "cls/predictions/transform/LayerNorm/gamma" } } } } node { name: "checkpoint_initializer_201/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_201" op: "RestoreV2" input: "checkpoint_initializer_201/prefix" input: "checkpoint_initializer_201/tensor_names" input: "checkpoint_initializer_201/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_201" op: "Assign" input: "cls/predictions/transform/LayerNorm/gamma" input: "checkpoint_initializer_201" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_202/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_202/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "cls/predictions/transform/dense/bias" } } } } node { name: "checkpoint_initializer_202/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_202" op: "RestoreV2" input: "checkpoint_initializer_202/prefix" input: "checkpoint_initializer_202/tensor_names" input: "checkpoint_initializer_202/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_202" op: "Assign" input: "cls/predictions/transform/dense/bias" input: "checkpoint_initializer_202" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_203/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_203/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "cls/predictions/transform/dense/kernel" } } } } node { name: "checkpoint_initializer_203/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_203" op: "RestoreV2" input: "checkpoint_initializer_203/prefix" input: "checkpoint_initializer_203/tensor_names" input: "checkpoint_initializer_203/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_203" op: "Assign" input: "cls/predictions/transform/dense/kernel" input: "checkpoint_initializer_203" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_204/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_204/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "cls/seq_relationship/output_bias" } } } } node { name: "checkpoint_initializer_204/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_204" op: "RestoreV2" input: "checkpoint_initializer_204/prefix" input: "checkpoint_initializer_204/tensor_names" input: "checkpoint_initializer_204/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_204" op: "Assign" input: "cls/seq_relationship/output_bias" input: "checkpoint_initializer_204" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "checkpoint_initializer_205/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/home/ema30/raid1/clinical_BERT/clinical_BERT/models/biobert/pubmed_pmc_470k/biobert_model.ckpt" } } } } node { name: "checkpoint_initializer_205/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "cls/seq_relationship/output_weights" } } } } node { name: "checkpoint_initializer_205/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_205" op: "RestoreV2" input: "checkpoint_initializer_205/prefix" input: "checkpoint_initializer_205/tensor_names" input: "checkpoint_initializer_205/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Assign_205" op: "Assign" input: "cls/seq_relationship/output_weights" input: "checkpoint_initializer_205" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_weights" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 4.999999873689376e-05 } } } } node { name: "PolynomialDecay/Cast/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "PolynomialDecay/Cast_1/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "PolynomialDecay/Cast_2/ReadVariableOp" op: "ReadVariableOp" input: "global_step" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT64 } } } node { name: "PolynomialDecay/Cast_2" op: "Cast" input: "PolynomialDecay/Cast_2/ReadVariableOp" attr { key: "DstT" value { type: DT_FLOAT } } attr { key: "SrcT" value { type: DT_INT64 } } attr { key: "Truncate" value { b: false } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "PolynomialDecay/Cast_3/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 100000 } } } } node { name: "PolynomialDecay/Cast_3" op: "Cast" input: "PolynomialDecay/Cast_3/x" attr { key: "DstT" value { type: DT_FLOAT } } attr { key: "SrcT" value { type: DT_INT32 } } attr { key: "Truncate" value { b: false } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "PolynomialDecay/Minimum/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 100000.0 } } } } node { name: "PolynomialDecay/Minimum" op: "Minimum" input: "PolynomialDecay/Cast_2" input: "PolynomialDecay/Minimum/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "PolynomialDecay/div" op: "RealDiv" input: "PolynomialDecay/Minimum" input: "PolynomialDecay/Cast_3" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "PolynomialDecay/sub" op: "Sub" input: "Const_1" input: "PolynomialDecay/Cast/x" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "PolynomialDecay/sub_1/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "PolynomialDecay/sub_1" op: "Sub" input: "PolynomialDecay/sub_1/x" input: "PolynomialDecay/div" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "PolynomialDecay/Pow" op: "Pow" input: "PolynomialDecay/sub_1" input: "PolynomialDecay/Cast_1/x" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "PolynomialDecay/Mul" op: "Mul" input: "PolynomialDecay/sub" input: "PolynomialDecay/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "PolynomialDecay" op: "Add" input: "PolynomialDecay/Mul" input: "PolynomialDecay/Cast/x" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "Cast/ReadVariableOp" op: "ReadVariableOp" input: "global_step" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT64 } } } node { name: "Cast" op: "Cast" input: "Cast/ReadVariableOp" attr { key: "DstT" value { type: DT_INT32 } } attr { key: "SrcT" value { type: DT_INT64 } } attr { key: "Truncate" value { b: false } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "Const_2" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 10000 } } } } node { name: "Cast_1" op: "Cast" input: "Cast" attr { key: "DstT" value { type: DT_FLOAT } } attr { key: "SrcT" value { type: DT_INT32 } } attr { key: "Truncate" value { b: false } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "Cast_2" op: "Cast" input: "Const_2" attr { key: "DstT" value { type: DT_FLOAT } } attr { key: "SrcT" value { type: DT_INT32 } } attr { key: "Truncate" value { b: false } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "truediv" op: "RealDiv" input: "Cast_1" input: "Cast_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "mul_1/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 4.999999873689376e-05 } } } } node { name: "mul_1" op: "Mul" input: "mul_1/x" input: "truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "Less" op: "Less" input: "Cast" input: "Const_2" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "Cast_3" op: "Cast" input: "Less" attr { key: "DstT" value { type: DT_FLOAT } } attr { key: "SrcT" value { type: DT_BOOL } } attr { key: "Truncate" value { b: false } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "sub/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "sub" op: "Sub" input: "sub/x" input: "Cast_3" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "mul_2" op: "Mul" input: "sub" input: "PolynomialDecay" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "mul_3" op: "Mul" input: "Cast_3" input: "mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "add_2" op: "Add" input: "mul_2" input: "mul_3" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/grad_ys_0" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "gradients/Fill" op: "Fill" input: "gradients/Shape" input: "gradients/grad_ys_0" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/cls/predictions/truediv_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/cls/predictions/truediv_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/cls/predictions/truediv_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/cls/predictions/truediv_grad/Shape" input: "gradients/cls/predictions/truediv_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/cls/predictions/truediv_grad/RealDiv" op: "RealDiv" input: "gradients/Fill" input: "cls/predictions/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/cls/predictions/truediv_grad/Sum" op: "Sum" input: "gradients/cls/predictions/truediv_grad/RealDiv" input: "gradients/cls/predictions/truediv_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/cls/predictions/truediv_grad/Reshape" op: "Reshape" input: "gradients/cls/predictions/truediv_grad/Sum" input: "gradients/cls/predictions/truediv_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/cls/predictions/truediv_grad/Neg" op: "Neg" input: "cls/predictions/Sum_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/cls/predictions/truediv_grad/RealDiv_1" op: "RealDiv" input: "gradients/cls/predictions/truediv_grad/Neg" input: "cls/predictions/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/cls/predictions/truediv_grad/RealDiv_2" op: "RealDiv" input: "gradients/cls/predictions/truediv_grad/RealDiv_1" input: "cls/predictions/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/cls/predictions/truediv_grad/mul" op: "Mul" input: "gradients/Fill" input: "gradients/cls/predictions/truediv_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/cls/predictions/truediv_grad/Sum_1" op: "Sum" input: "gradients/cls/predictions/truediv_grad/mul" input: "gradients/cls/predictions/truediv_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/cls/predictions/truediv_grad/Reshape_1" op: "Reshape" input: "gradients/cls/predictions/truediv_grad/Sum_1" input: "gradients/cls/predictions/truediv_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/cls/seq_relationship/Mean_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/cls/seq_relationship/Mean_grad/Reshape" op: "Reshape" input: "gradients/Fill" input: "gradients/cls/seq_relationship/Mean_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/cls/seq_relationship/Mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 32 } } } } node { name: "gradients/cls/seq_relationship/Mean_grad/Tile" op: "Tile" input: "gradients/cls/seq_relationship/Mean_grad/Reshape" input: "gradients/cls/seq_relationship/Mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } } } } } } node { name: "gradients/cls/seq_relationship/Mean_grad/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 32.0 } } } } node { name: "gradients/cls/seq_relationship/Mean_grad/truediv" op: "RealDiv" input: "gradients/cls/seq_relationship/Mean_grad/Tile" input: "gradients/cls/seq_relationship/Mean_grad/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } } } } } } node { name: "gradients/cls/predictions/Sum_1_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/cls/predictions/Sum_1_grad/Reshape" op: "Reshape" input: "gradients/cls/predictions/truediv_grad/Reshape" input: "gradients/cls/predictions/Sum_1_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/cls/predictions/Sum_1_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 640 } } } } node { name: "gradients/cls/predictions/Sum_1_grad/Tile" op: "Tile" input: "gradients/cls/predictions/Sum_1_grad/Reshape" input: "gradients/cls/predictions/Sum_1_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } } } } } } node { name: "gradients/cls/seq_relationship/Neg_grad/Neg" op: "Neg" input: "gradients/cls/seq_relationship/Mean_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } } } } } } node { name: "gradients/cls/predictions/mul_1_grad/Mul" op: "Mul" input: "gradients/cls/predictions/Sum_1_grad/Tile" input: "cls/predictions/Neg" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } } } } } } node { name: "gradients/cls/predictions/mul_1_grad/Mul_1" op: "Mul" input: "gradients/cls/predictions/Sum_1_grad/Tile" input: "cls/predictions/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } } } } } } node { name: "gradients/cls/seq_relationship/Sum_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: " \000\000\000\002\000\000\000" } } } } node { name: "gradients/cls/seq_relationship/Sum_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/cls/seq_relationship/Sum_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/cls/seq_relationship/Sum_grad/add" op: "Add" input: "cls/seq_relationship/Sum/reduction_indices" input: "gradients/cls/seq_relationship/Sum_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/cls/seq_relationship/Sum_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/cls/seq_relationship/Sum_grad/mod" op: "FloorMod" input: "gradients/cls/seq_relationship/Sum_grad/add" input: "gradients/cls/seq_relationship/Sum_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/cls/seq_relationship/Sum_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/cls/seq_relationship/Sum_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/cls/seq_relationship/Sum_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/cls/seq_relationship/Sum_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/cls/seq_relationship/Sum_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/cls/seq_relationship/Sum_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/cls/seq_relationship/Sum_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/cls/seq_relationship/Sum_grad/range" op: "Range" input: "gradients/cls/seq_relationship/Sum_grad/range/start" input: "gradients/cls/seq_relationship/Sum_grad/Size" input: "gradients/cls/seq_relationship/Sum_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/cls/seq_relationship/Sum_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/cls/seq_relationship/Sum_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/cls/seq_relationship/Sum_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/cls/seq_relationship/Sum_grad/Fill" op: "Fill" input: "gradients/cls/seq_relationship/Sum_grad/Shape_1" input: "gradients/cls/seq_relationship/Sum_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/cls/seq_relationship/Sum_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/cls/seq_relationship/Sum_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/cls/seq_relationship/Sum_grad/range" input: "gradients/cls/seq_relationship/Sum_grad/mod" input: "gradients/cls/seq_relationship/Sum_grad/Shape" input: "gradients/cls/seq_relationship/Sum_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/cls/seq_relationship/Sum_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/cls/seq_relationship/Sum_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/cls/seq_relationship/Sum_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/cls/seq_relationship/Sum_grad/Maximum" op: "Maximum" input: "gradients/cls/seq_relationship/Sum_grad/DynamicStitch" input: "gradients/cls/seq_relationship/Sum_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/cls/seq_relationship/Sum_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/cls/seq_relationship/Sum_grad/floordiv" op: "FloorDiv" input: "gradients/cls/seq_relationship/Sum_grad/Shape" input: "gradients/cls/seq_relationship/Sum_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/cls/seq_relationship/Sum_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/cls/seq_relationship/Sum_grad/Reshape" op: "Reshape" input: "gradients/cls/seq_relationship/Neg_grad/Neg" input: "gradients/cls/seq_relationship/Sum_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } } } } } } node { name: "gradients/cls/seq_relationship/Sum_grad/Tile" op: "Tile" input: "gradients/cls/seq_relationship/Sum_grad/Reshape" input: "gradients/cls/seq_relationship/Sum_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 2 } } } } } } node { name: "gradients/cls/predictions/Neg_grad/Neg" op: "Neg" input: "gradients/cls/predictions/mul_1_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } } } } } } node { name: "gradients/cls/seq_relationship/mul_grad/Mul" op: "Mul" input: "gradients/cls/seq_relationship/Sum_grad/Tile" input: "cls/seq_relationship/LogSoftmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 2 } } } } } } node { name: "gradients/cls/seq_relationship/mul_grad/Mul_1" op: "Mul" input: "gradients/cls/seq_relationship/Sum_grad/Tile" input: "cls/seq_relationship/one_hot" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 2 } } } } } } node { name: "gradients/cls/predictions/Sum_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\200\002\000\000Dq\000\000" } } } } node { name: "gradients/cls/predictions/Sum_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/Sum_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/cls/predictions/Sum_grad/add" op: "Add" input: "cls/predictions/Sum/reduction_indices" input: "gradients/cls/predictions/Sum_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/Sum_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/cls/predictions/Sum_grad/mod" op: "FloorMod" input: "gradients/cls/predictions/Sum_grad/add" input: "gradients/cls/predictions/Sum_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/Sum_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/cls/predictions/Sum_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/Sum_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/cls/predictions/Sum_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/Sum_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/cls/predictions/Sum_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/Sum_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/cls/predictions/Sum_grad/range" op: "Range" input: "gradients/cls/predictions/Sum_grad/range/start" input: "gradients/cls/predictions/Sum_grad/Size" input: "gradients/cls/predictions/Sum_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/Sum_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/cls/predictions/Sum_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/Sum_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/cls/predictions/Sum_grad/Fill" op: "Fill" input: "gradients/cls/predictions/Sum_grad/Shape_1" input: "gradients/cls/predictions/Sum_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/Sum_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/cls/predictions/Sum_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/cls/predictions/Sum_grad/range" input: "gradients/cls/predictions/Sum_grad/mod" input: "gradients/cls/predictions/Sum_grad/Shape" input: "gradients/cls/predictions/Sum_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/Sum_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/cls/predictions/Sum_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/Sum_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/cls/predictions/Sum_grad/Maximum" op: "Maximum" input: "gradients/cls/predictions/Sum_grad/DynamicStitch" input: "gradients/cls/predictions/Sum_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/Sum_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/cls/predictions/Sum_grad/floordiv" op: "FloorDiv" input: "gradients/cls/predictions/Sum_grad/Shape" input: "gradients/cls/predictions/Sum_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/Sum_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/cls/predictions/Sum_grad/Reshape" op: "Reshape" input: "gradients/cls/predictions/Neg_grad/Neg" input: "gradients/cls/predictions/Sum_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 1 } } } } } } node { name: "gradients/cls/predictions/Sum_grad/Tile" op: "Tile" input: "gradients/cls/predictions/Sum_grad/Reshape" input: "gradients/cls/predictions/Sum_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 28996 } } } } } } node { name: "gradients/cls/seq_relationship/LogSoftmax_grad/Exp" op: "Exp" input: "cls/seq_relationship/LogSoftmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 2 } } } } } } node { name: "gradients/cls/seq_relationship/LogSoftmax_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: -1 } } } } node { name: "gradients/cls/seq_relationship/LogSoftmax_grad/Sum" op: "Sum" input: "gradients/cls/seq_relationship/mul_grad/Mul_1" input: "gradients/cls/seq_relationship/LogSoftmax_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/cls/seq_relationship/LogSoftmax_grad/mul" op: "Mul" input: "gradients/cls/seq_relationship/LogSoftmax_grad/Sum" input: "gradients/cls/seq_relationship/LogSoftmax_grad/Exp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 2 } } } } } } node { name: "gradients/cls/seq_relationship/LogSoftmax_grad/sub" op: "Sub" input: "gradients/cls/seq_relationship/mul_grad/Mul_1" input: "gradients/cls/seq_relationship/LogSoftmax_grad/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 2 } } } } } } node { name: "gradients/cls/predictions/mul_grad/Mul" op: "Mul" input: "gradients/cls/predictions/Sum_grad/Tile" input: "cls/predictions/one_hot" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 28996 } } } } } } node { name: "gradients/cls/predictions/mul_grad/Mul_1" op: "Mul" input: "gradients/cls/predictions/Sum_grad/Tile" input: "cls/predictions/LogSoftmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 28996 } } } } } } node { name: "gradients/cls/seq_relationship/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/cls/seq_relationship/LogSoftmax_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/cls/predictions/LogSoftmax_grad/Exp" op: "Exp" input: "cls/predictions/LogSoftmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 28996 } } } } } } node { name: "gradients/cls/predictions/LogSoftmax_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: -1 } } } } node { name: "gradients/cls/predictions/LogSoftmax_grad/Sum" op: "Sum" input: "gradients/cls/predictions/mul_grad/Mul" input: "gradients/cls/predictions/LogSoftmax_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/cls/predictions/LogSoftmax_grad/mul" op: "Mul" input: "gradients/cls/predictions/LogSoftmax_grad/Sum" input: "gradients/cls/predictions/LogSoftmax_grad/Exp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 28996 } } } } } } node { name: "gradients/cls/predictions/LogSoftmax_grad/sub" op: "Sub" input: "gradients/cls/predictions/mul_grad/Mul" input: "gradients/cls/predictions/LogSoftmax_grad/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 28996 } } } } } } node { name: "gradients/cls/seq_relationship/MatMul_grad/MatMul" op: "MatMul" input: "gradients/cls/seq_relationship/LogSoftmax_grad/sub" input: "cls/seq_relationship/output_weights/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/cls/seq_relationship/MatMul_grad/MatMul_1" op: "MatMul" input: "gradients/cls/seq_relationship/LogSoftmax_grad/sub" input: "bert/pooler/dense/Tanh" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/cls/predictions/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/cls/predictions/LogSoftmax_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/pooler/dense/Tanh_grad/TanhGrad" op: "TanhGrad" input: "bert/pooler/dense/Tanh" input: "gradients/cls/seq_relationship/MatMul_grad/MatMul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 768 } } } } } } node { name: "gradients/cls/predictions/MatMul_grad/MatMul" op: "MatMul" input: "gradients/cls/predictions/LogSoftmax_grad/sub" input: "bert/embeddings/word_embeddings/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/cls/predictions/MatMul_grad/MatMul_1" op: "MatMul" input: "gradients/cls/predictions/LogSoftmax_grad/sub" input: "cls/predictions/transform/LayerNorm/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/pooler/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/pooler/dense/Tanh_grad/TanhGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/pooler/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/pooler/dense/Tanh_grad/TanhGrad" input: "bert/pooler/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/pooler/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/pooler/Squeeze" input: "gradients/bert/pooler/dense/Tanh_grad/TanhGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/cls/predictions/MatMul_grad/MatMul" input: "cls/predictions/transform/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/cls/predictions/MatMul_grad/MatMul" input: "cls/predictions/transform/dense/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/batchnorm/sub_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/batchnorm/sub_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\200\002\000\000\000\003\000\000" } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/sub_grad/Shape" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/cls/predictions/MatMul_grad/MatMul" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/cls/predictions/transform/LayerNorm/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/sub_grad/Sum" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/sub_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/batchnorm/sub_grad/Sum_1" op: "Sum" input: "gradients/cls/predictions/MatMul_grad/MatMul" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/cls/predictions/transform/LayerNorm/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/sub_grad/Sum_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/batchnorm/sub_grad/Reshape_1" op: "Reshape" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/sub_grad/Neg" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "gradients/bert/pooler/Squeeze_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: " \000\000\000\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/pooler/Squeeze_grad/Reshape" op: "Reshape" input: "gradients/bert/pooler/dense/MatMul_grad/MatMul" input: "gradients/bert/pooler/Squeeze_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 768 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\200\002\000\000\001\000\000\000" } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\200\002\000\000\000\003\000\000" } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_2_grad/Shape" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/sub_grad/Reshape_1" input: "cls/predictions/transform/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_2_grad/Mul" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_2_grad/Sum" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 1 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "cls/predictions/transform/LayerNorm/moments/mean" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/sub_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_2_grad/Sum_1" op: "Sum" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_2_grad/Mul_1" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_2_grad/Reshape_1" op: "Reshape" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_2_grad/Sum_1" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "gradients/bert/pooler/strided_slice_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: " \000\000\000\200\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/pooler/strided_slice_grad/StridedSliceGrad" op: "StridedSliceGrad" input: "gradients/bert/pooler/strided_slice_grad/Shape" input: "bert/pooler/strided_slice/stack" input: "bert/pooler/strided_slice/stack_1" input: "bert/pooler/strided_slice/stack_2" input: "gradients/bert/pooler/Squeeze_grad/Reshape" attr { key: "Index" value { type: DT_INT32 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } attr { key: "begin_mask" value { i: 5 } } attr { key: "ellipsis_mask" value { i: 0 } } attr { key: "end_mask" value { i: 5 } } attr { key: "new_axis_mask" value { i: 0 } } attr { key: "shrink_axis_mask" value { i: 0 } } } node { name: "gradients/AddN" op: "AddN" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_1_grad/Mul_1" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_2_grad/Reshape_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\200\002\000\000\001\000\000\000" } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_grad/Shape" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN" input: "cls/predictions/transform/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_grad/Mul" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_grad/Sum" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 1 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_grad/Mul_1" op: "Mul" input: "cls/predictions/transform/LayerNorm/batchnorm/Rsqrt" input: "gradients/AddN" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_grad/Mul_1" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_grad/Sum_1" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "cls/predictions/transform/LayerNorm/batchnorm/Rsqrt" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 1 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/batchnorm/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\200\002\000\000\001\000\000\000" } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/batchnorm/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/add_grad/Shape" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/cls/predictions/transform/LayerNorm/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/add_grad/Sum" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 1 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/batchnorm/add_grad/Sum_1" op: "Sum" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/cls/predictions/transform/LayerNorm/batchnorm/add_grad/Reshape_1" op: "Reshape" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/add_grad/Sum_1" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\200\002\000\000\000\003\000\000" } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/add" op: "Add" input: "cls/predictions/transform/LayerNorm/moments/variance/reduction_indices" input: "gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/mod" op: "FloorMod" input: "gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/add" input: "gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/range" op: "Range" input: "gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/range/start" input: "gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/Size" input: "gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/Fill" op: "Fill" input: "gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/Shape_1" input: "gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/range" input: "gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/mod" input: "gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/Shape" input: "gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/Maximum" op: "Maximum" input: "gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/DynamicStitch" input: "gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/floordiv" op: "FloorDiv" input: "gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/Shape" input: "gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/add_grad/Reshape" input: "gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 1 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/Tile" op: "Tile" input: "gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/Reshape" input: "gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/Tile" input: "gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/SquaredDifference_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\200\002\000\000\000\003\000\000" } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/SquaredDifference_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\200\002\000\000\001\000\000\000" } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/cls/predictions/transform/LayerNorm/moments/SquaredDifference_grad/Shape" input: "gradients/cls/predictions/transform/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/SquaredDifference_grad/mul" op: "Mul" input: "gradients/cls/predictions/transform/LayerNorm/moments/SquaredDifference_grad/scalar" input: "gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/SquaredDifference_grad/sub" op: "Sub" input: "cls/predictions/transform/dense/mul_1" input: "cls/predictions/transform/LayerNorm/moments/StopGradient" input: "^gradients/cls/predictions/transform/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/cls/predictions/transform/LayerNorm/moments/SquaredDifference_grad/mul" input: "gradients/cls/predictions/transform/LayerNorm/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/cls/predictions/transform/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/cls/predictions/transform/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/cls/predictions/transform/LayerNorm/moments/SquaredDifference_grad/Sum" input: "gradients/cls/predictions/transform/LayerNorm/moments/SquaredDifference_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/SquaredDifference_grad/Sum_1" op: "Sum" input: "gradients/cls/predictions/transform/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/cls/predictions/transform/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/SquaredDifference_grad/Reshape_1" op: "Reshape" input: "gradients/cls/predictions/transform/LayerNorm/moments/SquaredDifference_grad/Sum_1" input: "gradients/cls/predictions/transform/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 1 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/cls/predictions/transform/LayerNorm/moments/SquaredDifference_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 1 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\200\002\000\000\000\003\000\000" } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/add" op: "Add" input: "cls/predictions/transform/LayerNorm/moments/mean/reduction_indices" input: "gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/mod" op: "FloorMod" input: "gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/add" input: "gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/range" op: "Range" input: "gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/range/start" input: "gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/Size" input: "gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/Fill" op: "Fill" input: "gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/Shape_1" input: "gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/range" input: "gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/mod" input: "gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/Shape" input: "gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/Maximum" op: "Maximum" input: "gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/DynamicStitch" input: "gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/floordiv" op: "FloorDiv" input: "gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/Shape" input: "gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_2_grad/Reshape" input: "gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 1 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/Tile" op: "Tile" input: "gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/Reshape" input: "gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/Tile" input: "gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "gradients/AddN_1" op: "AddN" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_1_grad/Mul" input: "gradients/cls/predictions/transform/LayerNorm/moments/SquaredDifference_grad/Reshape" input: "gradients/cls/predictions/transform/LayerNorm/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "gradients/cls/predictions/transform/dense/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_1" input: "cls/predictions/transform/dense/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "gradients/cls/predictions/transform/dense/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_1" input: "cls/predictions/transform/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "gradients/cls/predictions/transform/dense/mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/cls/predictions/transform/dense/mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\200\002\000\000\000\003\000\000" } } } } node { name: "gradients/cls/predictions/transform/dense/mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/cls/predictions/transform/dense/mul_grad/Shape" input: "gradients/cls/predictions/transform/dense/mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/cls/predictions/transform/dense/mul_grad/Mul" op: "Mul" input: "gradients/cls/predictions/transform/dense/mul_1_grad/Mul_1" input: "cls/predictions/transform/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "gradients/cls/predictions/transform/dense/mul_grad/Sum" op: "Sum" input: "gradients/cls/predictions/transform/dense/mul_grad/Mul" input: "gradients/cls/predictions/transform/dense/mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/cls/predictions/transform/dense/mul_grad/Reshape" op: "Reshape" input: "gradients/cls/predictions/transform/dense/mul_grad/Sum" input: "gradients/cls/predictions/transform/dense/mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/cls/predictions/transform/dense/mul_grad/Mul_1" op: "Mul" input: "cls/predictions/transform/dense/mul/x" input: "gradients/cls/predictions/transform/dense/mul_1_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "gradients/cls/predictions/transform/dense/mul_grad/Sum_1" op: "Sum" input: "gradients/cls/predictions/transform/dense/mul_grad/Mul_1" input: "gradients/cls/predictions/transform/dense/mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/cls/predictions/transform/dense/mul_grad/Reshape_1" op: "Reshape" input: "gradients/cls/predictions/transform/dense/mul_grad/Sum_1" input: "gradients/cls/predictions/transform/dense/mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "gradients/cls/predictions/transform/dense/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/cls/predictions/transform/dense/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\200\002\000\000\000\003\000\000" } } } } node { name: "gradients/cls/predictions/transform/dense/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/cls/predictions/transform/dense/add_grad/Shape" input: "gradients/cls/predictions/transform/dense/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/cls/predictions/transform/dense/add_grad/Sum" op: "Sum" input: "gradients/cls/predictions/transform/dense/mul_grad/Reshape_1" input: "gradients/cls/predictions/transform/dense/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/cls/predictions/transform/dense/add_grad/Reshape" op: "Reshape" input: "gradients/cls/predictions/transform/dense/add_grad/Sum" input: "gradients/cls/predictions/transform/dense/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/cls/predictions/transform/dense/add_grad/Sum_1" op: "Sum" input: "gradients/cls/predictions/transform/dense/mul_grad/Reshape_1" input: "gradients/cls/predictions/transform/dense/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/cls/predictions/transform/dense/add_grad/Reshape_1" op: "Reshape" input: "gradients/cls/predictions/transform/dense/add_grad/Sum_1" input: "gradients/cls/predictions/transform/dense/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "gradients/cls/predictions/transform/dense/Erf_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.128379225730896 } } } } node { name: "gradients/cls/predictions/transform/dense/Erf_grad/mul" op: "Mul" input: "gradients/cls/predictions/transform/dense/add_grad/Reshape_1" input: "gradients/cls/predictions/transform/dense/Erf_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "gradients/cls/predictions/transform/dense/Erf_grad/Square" op: "Square" input: "cls/predictions/transform/dense/truediv" input: "^gradients/cls/predictions/transform/dense/add_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "gradients/cls/predictions/transform/dense/Erf_grad/Neg" op: "Neg" input: "gradients/cls/predictions/transform/dense/Erf_grad/Square" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "gradients/cls/predictions/transform/dense/Erf_grad/Exp" op: "Exp" input: "gradients/cls/predictions/transform/dense/Erf_grad/Neg" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "gradients/cls/predictions/transform/dense/Erf_grad/mul_1" op: "Mul" input: "gradients/cls/predictions/transform/dense/Erf_grad/mul" input: "gradients/cls/predictions/transform/dense/Erf_grad/Exp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "gradients/cls/predictions/transform/dense/truediv_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\200\002\000\000\000\003\000\000" } } } } node { name: "gradients/cls/predictions/transform/dense/truediv_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/cls/predictions/transform/dense/truediv_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/cls/predictions/transform/dense/truediv_grad/Shape" input: "gradients/cls/predictions/transform/dense/truediv_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/cls/predictions/transform/dense/truediv_grad/RealDiv" op: "RealDiv" input: "gradients/cls/predictions/transform/dense/Erf_grad/mul_1" input: "cls/predictions/transform/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "gradients/cls/predictions/transform/dense/truediv_grad/Sum" op: "Sum" input: "gradients/cls/predictions/transform/dense/truediv_grad/RealDiv" input: "gradients/cls/predictions/transform/dense/truediv_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/cls/predictions/transform/dense/truediv_grad/Reshape" op: "Reshape" input: "gradients/cls/predictions/transform/dense/truediv_grad/Sum" input: "gradients/cls/predictions/transform/dense/truediv_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "gradients/cls/predictions/transform/dense/truediv_grad/Neg" op: "Neg" input: "cls/predictions/transform/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "gradients/cls/predictions/transform/dense/truediv_grad/RealDiv_1" op: "RealDiv" input: "gradients/cls/predictions/transform/dense/truediv_grad/Neg" input: "cls/predictions/transform/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "gradients/cls/predictions/transform/dense/truediv_grad/RealDiv_2" op: "RealDiv" input: "gradients/cls/predictions/transform/dense/truediv_grad/RealDiv_1" input: "cls/predictions/transform/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "gradients/cls/predictions/transform/dense/truediv_grad/mul" op: "Mul" input: "gradients/cls/predictions/transform/dense/Erf_grad/mul_1" input: "gradients/cls/predictions/transform/dense/truediv_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "gradients/cls/predictions/transform/dense/truediv_grad/Sum_1" op: "Sum" input: "gradients/cls/predictions/transform/dense/truediv_grad/mul" input: "gradients/cls/predictions/transform/dense/truediv_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/cls/predictions/transform/dense/truediv_grad/Reshape_1" op: "Reshape" input: "gradients/cls/predictions/transform/dense/truediv_grad/Sum_1" input: "gradients/cls/predictions/transform/dense/truediv_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/AddN_2" op: "AddN" input: "gradients/cls/predictions/transform/dense/mul_1_grad/Mul" input: "gradients/cls/predictions/transform/dense/truediv_grad/Reshape" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/transform/dense/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "gradients/cls/predictions/transform/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/AddN_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/cls/predictions/transform/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/AddN_2" input: "cls/predictions/transform/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/cls/predictions/transform/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "GatherV2" input: "gradients/AddN_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/GatherV2_grad/Shape" op: "Const" attr { key: "_class" value { list { s: "loc:@Reshape_2" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT64 } } attr { key: "value" value { tensor { dtype: DT_INT64 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\000\000\000\000\003\000\000\000\000\000\000" } } } } node { name: "gradients/GatherV2_grad/ToInt32" op: "Cast" input: "gradients/GatherV2_grad/Shape" attr { key: "DstT" value { type: DT_INT32 } } attr { key: "SrcT" value { type: DT_INT64 } } attr { key: "Truncate" value { b: false } } attr { key: "_class" value { list { s: "loc:@Reshape_2" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/GatherV2_grad/Size" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 640 } } } } node { name: "gradients/GatherV2_grad/ExpandDims/dim" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/GatherV2_grad/ExpandDims" op: "ExpandDims" input: "gradients/GatherV2_grad/Size" input: "gradients/GatherV2_grad/ExpandDims/dim" attr { key: "T" value { type: DT_INT32 } } attr { key: "Tdim" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/GatherV2_grad/strided_slice/stack" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/GatherV2_grad/strided_slice/stack_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/GatherV2_grad/strided_slice/stack_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/GatherV2_grad/strided_slice" op: "StridedSlice" input: "gradients/GatherV2_grad/ToInt32" input: "gradients/GatherV2_grad/strided_slice/stack" input: "gradients/GatherV2_grad/strided_slice/stack_1" input: "gradients/GatherV2_grad/strided_slice/stack_2" attr { key: "Index" value { type: DT_INT32 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "begin_mask" value { i: 0 } } attr { key: "ellipsis_mask" value { i: 0 } } attr { key: "end_mask" value { i: 1 } } attr { key: "new_axis_mask" value { i: 0 } } attr { key: "shrink_axis_mask" value { i: 0 } } } node { name: "gradients/GatherV2_grad/concat/axis" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/GatherV2_grad/concat" op: "ConcatV2" input: "gradients/GatherV2_grad/ExpandDims" input: "gradients/GatherV2_grad/strided_slice" input: "gradients/GatherV2_grad/concat/axis" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/GatherV2_grad/Reshape" op: "Reshape" input: "gradients/cls/predictions/transform/dense/MatMul_grad/MatMul" input: "gradients/GatherV2_grad/concat" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } dim { size: 768 } } } } } } node { name: "gradients/GatherV2_grad/Reshape_1" op: "Reshape" input: "Reshape_1" input: "gradients/GatherV2_grad/ExpandDims" attr { key: "T" value { type: DT_INT32 } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 640 } } } } } } node { name: "gradients/Reshape_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: " \000\000\000\200\000\000\000\000\003\000\000" } } } } node { name: "gradients/Reshape_2_grad/Reshape/strided_slice/stack" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/Reshape_2_grad/Reshape/strided_slice/stack_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/Reshape_2_grad/Reshape/strided_slice/stack_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/Reshape_2_grad/Reshape/strided_slice" op: "StridedSlice" input: "gradients/GatherV2_grad/ToInt32" input: "gradients/Reshape_2_grad/Reshape/strided_slice/stack" input: "gradients/Reshape_2_grad/Reshape/strided_slice/stack_1" input: "gradients/Reshape_2_grad/Reshape/strided_slice/stack_2" attr { key: "Index" value { type: DT_INT32 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "begin_mask" value { i: 0 } } attr { key: "ellipsis_mask" value { i: 0 } } attr { key: "end_mask" value { i: 0 } } attr { key: "new_axis_mask" value { i: 0 } } attr { key: "shrink_axis_mask" value { i: 1 } } } node { name: "gradients/Reshape_2_grad/Reshape/tensor" op: "UnsortedSegmentSum" input: "gradients/GatherV2_grad/Reshape" input: "gradients/GatherV2_grad/Reshape_1" input: "gradients/Reshape_2_grad/Reshape/strided_slice" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tindices" value { type: DT_INT32 } } attr { key: "Tnumsegments" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/Reshape_2_grad/Reshape" op: "Reshape" input: "gradients/Reshape_2_grad/Reshape/tensor" input: "gradients/Reshape_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "gradients/AddN_3" op: "AddN" input: "gradients/bert/pooler/strided_slice_grad/StridedSliceGrad" input: "gradients/Reshape_2_grad/Reshape" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/pooler/strided_slice_grad/StridedSliceGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/Reshape_13_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/Reshape_13_grad/Reshape" op: "Reshape" input: "gradients/AddN_3" input: "gradients/bert/encoder/Reshape_13_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/bert/encoder/Reshape_13_grad/Reshape" input: "bert/encoder/layer_11/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/bert/encoder/Reshape_13_grad/Reshape" input: "bert/encoder/layer_11/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Shape" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/bert/encoder/Reshape_13_grad/Reshape" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/Reshape_13_grad/Reshape" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Sum_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Neg" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Shape" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Reshape_1" input: "bert/encoder/layer_11/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_11/output/LayerNorm/moments/mean" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_4" op: "AddN" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Shape" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_4" input: "bert/encoder/layer_11/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_11/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/AddN_4" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_11/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/add_grad/Shape" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/add_grad/Sum_1" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/add" op: "Add" input: "bert/encoder/layer_11/output/LayerNorm/moments/variance/reduction_indices" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/add" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/range" op: "Range" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/range/start" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/Size" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/Shape_1" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/range" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/mod" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/DynamicStitch" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/add_grad/Reshape" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/Shape" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_11/output/add" input: "bert/encoder/layer_11/output/LayerNorm/moments/StopGradient" input: "^gradients/bert/encoder/layer_11/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/mul" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/add" op: "Add" input: "bert/encoder/layer_11/output/LayerNorm/moments/mean/reduction_indices" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/add" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/range" op: "Range" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/range/start" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/Size" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/Shape_1" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/range" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/mod" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/DynamicStitch" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_5" op: "AddN" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/Reshape" input: "gradients/bert/encoder/layer_11/output/LayerNorm/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/dropout/mul_grad/Mul" op: "Mul" input: "gradients/AddN_5" input: "bert/encoder/layer_11/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/dropout/mul_grad/Mul_1" op: "Mul" input: "gradients/AddN_5" input: "bert/encoder/layer_11/output/dropout/div" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/dropout/div_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/output/dropout/div_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_11/output/dropout/div_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_11/output/dropout/div_grad/Shape" input: "gradients/bert/encoder/layer_11/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/dropout/div_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_11/output/dropout/mul_grad/Mul" input: "bert/encoder/layer_11/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/dropout/div_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_11/output/dropout/div_grad/RealDiv" input: "gradients/bert/encoder/layer_11/output/dropout/div_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/output/dropout/div_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/output/dropout/div_grad/Sum" input: "gradients/bert/encoder/layer_11/output/dropout/div_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/dropout/div_grad/Neg" op: "Neg" input: "bert/encoder/layer_11/output/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/dropout/div_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_11/output/dropout/div_grad/Neg" input: "bert/encoder/layer_11/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/dropout/div_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_11/output/dropout/div_grad/RealDiv_1" input: "bert/encoder/layer_11/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/dropout/div_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_11/output/dropout/mul_grad/Mul" input: "gradients/bert/encoder/layer_11/output/dropout/div_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/dropout/div_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_11/output/dropout/div_grad/mul" input: "gradients/bert/encoder/layer_11/output/dropout/div_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/output/dropout/div_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_11/output/dropout/div_grad/Sum_1" input: "gradients/bert/encoder/layer_11/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_11/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_11/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_11/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_11/output/dropout/div_grad/Reshape" input: "bert/encoder/layer_11/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_11/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_11/intermediate/dense/mul_1" input: "gradients/bert/encoder/layer_11/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/mul_1_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_11/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_11/intermediate/dense/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/mul_1_grad/Mul_1" op: "Mul" input: "gradients/bert/encoder/layer_11/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_11/intermediate/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_11/intermediate/dense/mul_grad/Shape" input: "gradients/bert/encoder/layer_11/intermediate/dense/mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_11/intermediate/dense/mul_1_grad/Mul_1" input: "bert/encoder/layer_11/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_11/intermediate/dense/mul_grad/Mul" input: "gradients/bert/encoder/layer_11/intermediate/dense/mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/intermediate/dense/mul_grad/Sum" input: "gradients/bert/encoder/layer_11/intermediate/dense/mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_11/intermediate/dense/mul/x" input: "gradients/bert/encoder/layer_11/intermediate/dense/mul_1_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_11/intermediate/dense/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_11/intermediate/dense/mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_11/intermediate/dense/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_11/intermediate/dense/mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_11/intermediate/dense/add_grad/Shape" input: "gradients/bert/encoder/layer_11/intermediate/dense/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_11/intermediate/dense/mul_grad/Reshape_1" input: "gradients/bert/encoder/layer_11/intermediate/dense/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/intermediate/dense/add_grad/Sum" input: "gradients/bert/encoder/layer_11/intermediate/dense/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_11/intermediate/dense/mul_grad/Reshape_1" input: "gradients/bert/encoder/layer_11/intermediate/dense/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_11/intermediate/dense/add_grad/Sum_1" input: "gradients/bert/encoder/layer_11/intermediate/dense/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/Erf_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.128379225730896 } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/Erf_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_11/intermediate/dense/add_grad/Reshape_1" input: "gradients/bert/encoder/layer_11/intermediate/dense/Erf_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/Erf_grad/Square" op: "Square" input: "bert/encoder/layer_11/intermediate/dense/truediv" input: "^gradients/bert/encoder/layer_11/intermediate/dense/add_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/Erf_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_11/intermediate/dense/Erf_grad/Square" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/Erf_grad/Exp" op: "Exp" input: "gradients/bert/encoder/layer_11/intermediate/dense/Erf_grad/Neg" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/Erf_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_11/intermediate/dense/Erf_grad/mul" input: "gradients/bert/encoder/layer_11/intermediate/dense/Erf_grad/Exp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/truediv_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/truediv_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/truediv_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_11/intermediate/dense/truediv_grad/Shape" input: "gradients/bert/encoder/layer_11/intermediate/dense/truediv_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/truediv_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_11/intermediate/dense/Erf_grad/mul_1" input: "bert/encoder/layer_11/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/truediv_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_11/intermediate/dense/truediv_grad/RealDiv" input: "gradients/bert/encoder/layer_11/intermediate/dense/truediv_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/truediv_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/intermediate/dense/truediv_grad/Sum" input: "gradients/bert/encoder/layer_11/intermediate/dense/truediv_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/truediv_grad/Neg" op: "Neg" input: "bert/encoder/layer_11/intermediate/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/truediv_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_11/intermediate/dense/truediv_grad/Neg" input: "bert/encoder/layer_11/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/truediv_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_11/intermediate/dense/truediv_grad/RealDiv_1" input: "bert/encoder/layer_11/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/truediv_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_11/intermediate/dense/Erf_grad/mul_1" input: "gradients/bert/encoder/layer_11/intermediate/dense/truediv_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/truediv_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_11/intermediate/dense/truediv_grad/mul" input: "gradients/bert/encoder/layer_11/intermediate/dense/truediv_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/truediv_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_11/intermediate/dense/truediv_grad/Sum_1" input: "gradients/bert/encoder/layer_11/intermediate/dense/truediv_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/AddN_6" op: "AddN" input: "gradients/bert/encoder/layer_11/intermediate/dense/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_11/intermediate/dense/truediv_grad/Reshape" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/intermediate/dense/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/AddN_6" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/AddN_6" input: "bert/encoder/layer_11/intermediate/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_1" input: "gradients/AddN_6" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_7" op: "AddN" input: "gradients/AddN_5" input: "gradients/bert/encoder/layer_11/intermediate/dense/MatMul_grad/MatMul" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_7" input: "bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_7" input: "bert/encoder/layer_11/attention/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Shape" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_7" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Sum_1" op: "Sum" input: "gradients/AddN_7" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Sum_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Neg" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Reshape_1" input: "bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_11/attention/output/LayerNorm/moments/mean" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_8" op: "AddN" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Shape" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_8" input: "bert/encoder/layer_11/attention/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/AddN_8" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_grad/Shape" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_grad/Sum_1" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/add" op: "Add" input: "bert/encoder/layer_11/attention/output/LayerNorm/moments/variance/reduction_indices" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/add" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/range" op: "Range" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/range/start" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Size" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Shape_1" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/range" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/mod" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/DynamicStitch" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_grad/Reshape" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_11/attention/output/add" input: "bert/encoder/layer_11/attention/output/LayerNorm/moments/StopGradient" input: "^gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/mul" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/add" op: "Add" input: "bert/encoder/layer_11/attention/output/LayerNorm/moments/mean/reduction_indices" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/add" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/range" op: "Range" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/range/start" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Size" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Shape_1" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/range" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/mod" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/DynamicStitch" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_9" op: "AddN" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/dropout/mul_grad/Mul" op: "Mul" input: "gradients/AddN_9" input: "bert/encoder/layer_11/attention/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/dropout/mul_grad/Mul_1" op: "Mul" input: "gradients/AddN_9" input: "bert/encoder/layer_11/attention/output/dropout/div" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/dropout/div_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/dropout/div_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/dropout/div_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_11/attention/output/dropout/div_grad/Shape" input: "gradients/bert/encoder/layer_11/attention/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/dropout/div_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_11/attention/output/dropout/mul_grad/Mul" input: "bert/encoder/layer_11/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/dropout/div_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_11/attention/output/dropout/div_grad/RealDiv" input: "gradients/bert/encoder/layer_11/attention/output/dropout/div_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/attention/output/dropout/div_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/output/dropout/div_grad/Sum" input: "gradients/bert/encoder/layer_11/attention/output/dropout/div_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/dropout/div_grad/Neg" op: "Neg" input: "bert/encoder/layer_11/attention/output/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/dropout/div_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_11/attention/output/dropout/div_grad/Neg" input: "bert/encoder/layer_11/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/dropout/div_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_11/attention/output/dropout/div_grad/RealDiv_1" input: "bert/encoder/layer_11/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/dropout/div_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_11/attention/output/dropout/mul_grad/Mul" input: "gradients/bert/encoder/layer_11/attention/output/dropout/div_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/dropout/div_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_11/attention/output/dropout/div_grad/mul" input: "gradients/bert/encoder/layer_11/attention/output/dropout/div_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/attention/output/dropout/div_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/output/dropout/div_grad/Sum_1" input: "gradients/bert/encoder/layer_11/attention/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_11/attention/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_11/attention/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_11/attention/output/dropout/div_grad/Reshape" input: "bert/encoder/layer_11/attention/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_11/attention/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_11/attention/self/Reshape_3" input: "gradients/bert/encoder/layer_11/attention/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Reshape_3_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Reshape_3_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/output/dense/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_11/attention/self/Reshape_3_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/transpose_3_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_11/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/transpose_3_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_11/attention/self/Reshape_3_grad/Reshape" input: "gradients/bert/encoder/layer_11/attention/self/transpose_3_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/MatMul_1_grad/MatMul" op: "BatchMatMul" input: "gradients/bert/encoder/layer_11/attention/self/transpose_3_grad/transpose" input: "bert/encoder/layer_11/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "gradients/bert/encoder/layer_11/attention/self/MatMul_1_grad/MatMul_1" op: "BatchMatMul" input: "bert/encoder/layer_11/attention/self/dropout/mul" input: "gradients/bert/encoder/layer_11/attention/self/transpose_3_grad/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/attention/self/dropout/mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_11/attention/self/MatMul_1_grad/MatMul" input: "bert/encoder/layer_11/attention/self/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/dropout/mul_grad/Mul_1" op: "Mul" input: "gradients/bert/encoder/layer_11/attention/self/MatMul_1_grad/MatMul" input: "bert/encoder/layer_11/attention/self/dropout/div" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/transpose_2_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_11/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/transpose_2_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_11/attention/self/MatMul_1_grad/MatMul_1" input: "gradients/bert/encoder/layer_11/attention/self/transpose_2_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/dropout/div_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/dropout/div_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/dropout/div_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_11/attention/self/dropout/div_grad/Shape" input: "gradients/bert/encoder/layer_11/attention/self/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/dropout/div_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_11/attention/self/dropout/mul_grad/Mul" input: "bert/encoder/layer_11/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/dropout/div_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_11/attention/self/dropout/div_grad/RealDiv" input: "gradients/bert/encoder/layer_11/attention/self/dropout/div_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/attention/self/dropout/div_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/self/dropout/div_grad/Sum" input: "gradients/bert/encoder/layer_11/attention/self/dropout/div_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/dropout/div_grad/Neg" op: "Neg" input: "bert/encoder/layer_11/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/dropout/div_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_11/attention/self/dropout/div_grad/Neg" input: "bert/encoder/layer_11/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/dropout/div_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_11/attention/self/dropout/div_grad/RealDiv_1" input: "bert/encoder/layer_11/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/dropout/div_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_11/attention/self/dropout/mul_grad/Mul" input: "gradients/bert/encoder/layer_11/attention/self/dropout/div_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/dropout/div_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_11/attention/self/dropout/div_grad/mul" input: "gradients/bert/encoder/layer_11/attention/self/dropout/div_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/attention/self/dropout/div_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/self/dropout/div_grad/Sum_1" input: "gradients/bert/encoder/layer_11/attention/self/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Reshape_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Reshape_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/self/transpose_2_grad/transpose" input: "gradients/bert/encoder/layer_11/attention/self/Reshape_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Softmax_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_11/attention/self/dropout/div_grad/Reshape" input: "bert/encoder/layer_11/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Softmax_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: -1 } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Softmax_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_11/attention/self/Softmax_grad/mul" input: "gradients/bert/encoder/layer_11/attention/self/Softmax_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Softmax_grad/sub" op: "Sub" input: "gradients/bert/encoder/layer_11/attention/self/dropout/div_grad/Reshape" input: "gradients/bert/encoder/layer_11/attention/self/Softmax_grad/Sum" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Softmax_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_11/attention/self/Softmax_grad/sub" input: "bert/encoder/layer_11/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/value/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_11/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_11/attention/self/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\001\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_11/attention/self/add_grad/Shape" input: "gradients/bert/encoder/layer_11/attention/self/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_11/attention/self/Softmax_grad/mul_1" input: "gradients/bert/encoder/layer_11/attention/self/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/attention/self/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/self/add_grad/Sum" input: "gradients/bert/encoder/layer_11/attention/self/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_11/attention/self/Softmax_grad/mul_1" input: "gradients/bert/encoder/layer_11/attention/self/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/attention/self/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/self/add_grad/Sum_1" input: "gradients/bert/encoder/layer_11/attention/self/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/value/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_11/attention/self/Reshape_2_grad/Reshape" input: "bert/encoder/layer_11/attention/self/value/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_11/attention/self/value/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_10/output/LayerNorm/batchnorm/add_1" input: "gradients/bert/encoder/layer_11/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_11/attention/self/Mul_grad/Shape" input: "gradients/bert/encoder/layer_11/attention/self/Mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_11/attention/self/add_grad/Reshape" input: "bert/encoder/layer_11/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_11/attention/self/Mul_grad/Mul" input: "gradients/bert/encoder/layer_11/attention/self/Mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/self/Mul_grad/Sum" input: "gradients/bert/encoder/layer_11/attention/self/Mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_11/attention/self/MatMul" input: "gradients/bert/encoder/layer_11/attention/self/add_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_11/attention/self/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_11/attention/self/Mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/self/Mul_grad/Sum_1" input: "gradients/bert/encoder/layer_11/attention/self/Mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/MatMul_grad/MatMul" op: "BatchMatMul" input: "gradients/bert/encoder/layer_11/attention/self/Mul_grad/Reshape" input: "bert/encoder/layer_11/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/attention/self/MatMul_grad/MatMul_1" op: "BatchMatMul" input: "gradients/bert/encoder/layer_11/attention/self/Mul_grad/Reshape" input: "bert/encoder/layer_11/attention/self/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/attention/self/transpose_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_11/attention/self/transpose/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/transpose_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_11/attention/self/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_11/attention/self/transpose_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/transpose_1_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_11/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/transpose_1_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_11/attention/self/MatMul_grad/MatMul_1" input: "gradients/bert/encoder/layer_11/attention/self/transpose_1_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Reshape_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Reshape_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/self/transpose_grad/transpose" input: "gradients/bert/encoder/layer_11/attention/self/Reshape_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Reshape_1_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Reshape_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/self/transpose_1_grad/transpose" input: "gradients/bert/encoder/layer_11/attention/self/Reshape_1_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/query/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_11/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_11/attention/self/key/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_11/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_11/attention/self/query/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_11/attention/self/Reshape_grad/Reshape" input: "bert/encoder/layer_11/attention/self/query/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_11/attention/self/query/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_10/output/LayerNorm/batchnorm/add_1" input: "gradients/bert/encoder/layer_11/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/attention/self/key/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_11/attention/self/Reshape_1_grad/Reshape" input: "bert/encoder/layer_11/attention/self/key/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_11/attention/self/key/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_10/output/LayerNorm/batchnorm/add_1" input: "gradients/bert/encoder/layer_11/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_10" op: "AddN" input: "gradients/AddN_9" input: "gradients/bert/encoder/layer_11/attention/self/value/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_11/attention/self/query/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_11/attention/self/key/MatMul_grad/MatMul" attr { key: "N" value { i: 4 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_10" input: "bert/encoder/layer_10/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_10" input: "bert/encoder/layer_10/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Shape" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_10" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Sum_1" op: "Sum" input: "gradients/AddN_10" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Sum_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Neg" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Shape" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Reshape_1" input: "bert/encoder/layer_10/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_10/output/LayerNorm/moments/mean" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_11" op: "AddN" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Shape" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_11" input: "bert/encoder/layer_10/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_10/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/AddN_11" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_10/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/add_grad/Shape" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/add_grad/Sum_1" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/add" op: "Add" input: "bert/encoder/layer_10/output/LayerNorm/moments/variance/reduction_indices" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/add" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/range" op: "Range" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/range/start" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/Size" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/Shape_1" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/range" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/mod" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/DynamicStitch" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/add_grad/Reshape" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/Shape" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_10/output/add" input: "bert/encoder/layer_10/output/LayerNorm/moments/StopGradient" input: "^gradients/bert/encoder/layer_10/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/mul" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/add" op: "Add" input: "bert/encoder/layer_10/output/LayerNorm/moments/mean/reduction_indices" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/add" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/range" op: "Range" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/range/start" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/Size" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/Shape_1" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/range" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/mod" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/DynamicStitch" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_12" op: "AddN" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/Reshape" input: "gradients/bert/encoder/layer_10/output/LayerNorm/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/dropout/mul_grad/Mul" op: "Mul" input: "gradients/AddN_12" input: "bert/encoder/layer_10/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/dropout/mul_grad/Mul_1" op: "Mul" input: "gradients/AddN_12" input: "bert/encoder/layer_10/output/dropout/div" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/dropout/div_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/output/dropout/div_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_10/output/dropout/div_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_10/output/dropout/div_grad/Shape" input: "gradients/bert/encoder/layer_10/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/dropout/div_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_10/output/dropout/mul_grad/Mul" input: "bert/encoder/layer_10/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/dropout/div_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_10/output/dropout/div_grad/RealDiv" input: "gradients/bert/encoder/layer_10/output/dropout/div_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/output/dropout/div_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/output/dropout/div_grad/Sum" input: "gradients/bert/encoder/layer_10/output/dropout/div_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/dropout/div_grad/Neg" op: "Neg" input: "bert/encoder/layer_10/output/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/dropout/div_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_10/output/dropout/div_grad/Neg" input: "bert/encoder/layer_10/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/dropout/div_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_10/output/dropout/div_grad/RealDiv_1" input: "bert/encoder/layer_10/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/dropout/div_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_10/output/dropout/mul_grad/Mul" input: "gradients/bert/encoder/layer_10/output/dropout/div_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/dropout/div_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_10/output/dropout/div_grad/mul" input: "gradients/bert/encoder/layer_10/output/dropout/div_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/output/dropout/div_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_10/output/dropout/div_grad/Sum_1" input: "gradients/bert/encoder/layer_10/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_10/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_10/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_10/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_10/output/dropout/div_grad/Reshape" input: "bert/encoder/layer_10/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_10/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_10/intermediate/dense/mul_1" input: "gradients/bert/encoder/layer_10/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/mul_1_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_10/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_10/intermediate/dense/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/mul_1_grad/Mul_1" op: "Mul" input: "gradients/bert/encoder/layer_10/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_10/intermediate/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_10/intermediate/dense/mul_grad/Shape" input: "gradients/bert/encoder/layer_10/intermediate/dense/mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_10/intermediate/dense/mul_1_grad/Mul_1" input: "bert/encoder/layer_10/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_10/intermediate/dense/mul_grad/Mul" input: "gradients/bert/encoder/layer_10/intermediate/dense/mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/intermediate/dense/mul_grad/Sum" input: "gradients/bert/encoder/layer_10/intermediate/dense/mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_10/intermediate/dense/mul/x" input: "gradients/bert/encoder/layer_10/intermediate/dense/mul_1_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_10/intermediate/dense/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_10/intermediate/dense/mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_10/intermediate/dense/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_10/intermediate/dense/mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_10/intermediate/dense/add_grad/Shape" input: "gradients/bert/encoder/layer_10/intermediate/dense/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_10/intermediate/dense/mul_grad/Reshape_1" input: "gradients/bert/encoder/layer_10/intermediate/dense/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/intermediate/dense/add_grad/Sum" input: "gradients/bert/encoder/layer_10/intermediate/dense/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_10/intermediate/dense/mul_grad/Reshape_1" input: "gradients/bert/encoder/layer_10/intermediate/dense/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_10/intermediate/dense/add_grad/Sum_1" input: "gradients/bert/encoder/layer_10/intermediate/dense/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/Erf_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.128379225730896 } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/Erf_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_10/intermediate/dense/add_grad/Reshape_1" input: "gradients/bert/encoder/layer_10/intermediate/dense/Erf_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/Erf_grad/Square" op: "Square" input: "bert/encoder/layer_10/intermediate/dense/truediv" input: "^gradients/bert/encoder/layer_10/intermediate/dense/add_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/Erf_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_10/intermediate/dense/Erf_grad/Square" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/Erf_grad/Exp" op: "Exp" input: "gradients/bert/encoder/layer_10/intermediate/dense/Erf_grad/Neg" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/Erf_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_10/intermediate/dense/Erf_grad/mul" input: "gradients/bert/encoder/layer_10/intermediate/dense/Erf_grad/Exp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/truediv_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/truediv_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/truediv_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_10/intermediate/dense/truediv_grad/Shape" input: "gradients/bert/encoder/layer_10/intermediate/dense/truediv_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/truediv_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_10/intermediate/dense/Erf_grad/mul_1" input: "bert/encoder/layer_10/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/truediv_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_10/intermediate/dense/truediv_grad/RealDiv" input: "gradients/bert/encoder/layer_10/intermediate/dense/truediv_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/truediv_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/intermediate/dense/truediv_grad/Sum" input: "gradients/bert/encoder/layer_10/intermediate/dense/truediv_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/truediv_grad/Neg" op: "Neg" input: "bert/encoder/layer_10/intermediate/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/truediv_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_10/intermediate/dense/truediv_grad/Neg" input: "bert/encoder/layer_10/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/truediv_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_10/intermediate/dense/truediv_grad/RealDiv_1" input: "bert/encoder/layer_10/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/truediv_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_10/intermediate/dense/Erf_grad/mul_1" input: "gradients/bert/encoder/layer_10/intermediate/dense/truediv_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/truediv_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_10/intermediate/dense/truediv_grad/mul" input: "gradients/bert/encoder/layer_10/intermediate/dense/truediv_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/truediv_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_10/intermediate/dense/truediv_grad/Sum_1" input: "gradients/bert/encoder/layer_10/intermediate/dense/truediv_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/AddN_13" op: "AddN" input: "gradients/bert/encoder/layer_10/intermediate/dense/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_10/intermediate/dense/truediv_grad/Reshape" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/intermediate/dense/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/AddN_13" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/AddN_13" input: "bert/encoder/layer_10/intermediate/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_1" input: "gradients/AddN_13" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_14" op: "AddN" input: "gradients/AddN_12" input: "gradients/bert/encoder/layer_10/intermediate/dense/MatMul_grad/MatMul" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_14" input: "bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_14" input: "bert/encoder/layer_10/attention/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Shape" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_14" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Sum_1" op: "Sum" input: "gradients/AddN_14" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Sum_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Neg" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Reshape_1" input: "bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_10/attention/output/LayerNorm/moments/mean" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_15" op: "AddN" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Shape" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_15" input: "bert/encoder/layer_10/attention/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/AddN_15" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_grad/Shape" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_grad/Sum_1" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/add" op: "Add" input: "bert/encoder/layer_10/attention/output/LayerNorm/moments/variance/reduction_indices" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/add" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/range" op: "Range" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/range/start" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Size" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Shape_1" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/range" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/mod" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/DynamicStitch" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_grad/Reshape" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_10/attention/output/add" input: "bert/encoder/layer_10/attention/output/LayerNorm/moments/StopGradient" input: "^gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/mul" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/add" op: "Add" input: "bert/encoder/layer_10/attention/output/LayerNorm/moments/mean/reduction_indices" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/add" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/range" op: "Range" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/range/start" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Size" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Shape_1" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/range" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/mod" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/DynamicStitch" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_16" op: "AddN" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/dropout/mul_grad/Mul" op: "Mul" input: "gradients/AddN_16" input: "bert/encoder/layer_10/attention/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/dropout/mul_grad/Mul_1" op: "Mul" input: "gradients/AddN_16" input: "bert/encoder/layer_10/attention/output/dropout/div" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/dropout/div_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/dropout/div_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/dropout/div_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_10/attention/output/dropout/div_grad/Shape" input: "gradients/bert/encoder/layer_10/attention/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/dropout/div_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_10/attention/output/dropout/mul_grad/Mul" input: "bert/encoder/layer_10/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/dropout/div_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_10/attention/output/dropout/div_grad/RealDiv" input: "gradients/bert/encoder/layer_10/attention/output/dropout/div_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/attention/output/dropout/div_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/output/dropout/div_grad/Sum" input: "gradients/bert/encoder/layer_10/attention/output/dropout/div_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/dropout/div_grad/Neg" op: "Neg" input: "bert/encoder/layer_10/attention/output/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/dropout/div_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_10/attention/output/dropout/div_grad/Neg" input: "bert/encoder/layer_10/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/dropout/div_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_10/attention/output/dropout/div_grad/RealDiv_1" input: "bert/encoder/layer_10/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/dropout/div_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_10/attention/output/dropout/mul_grad/Mul" input: "gradients/bert/encoder/layer_10/attention/output/dropout/div_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/dropout/div_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_10/attention/output/dropout/div_grad/mul" input: "gradients/bert/encoder/layer_10/attention/output/dropout/div_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/attention/output/dropout/div_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/output/dropout/div_grad/Sum_1" input: "gradients/bert/encoder/layer_10/attention/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_10/attention/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_10/attention/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_10/attention/output/dropout/div_grad/Reshape" input: "bert/encoder/layer_10/attention/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_10/attention/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_10/attention/self/Reshape_3" input: "gradients/bert/encoder/layer_10/attention/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Reshape_3_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Reshape_3_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/output/dense/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_10/attention/self/Reshape_3_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/transpose_3_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_10/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/transpose_3_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_10/attention/self/Reshape_3_grad/Reshape" input: "gradients/bert/encoder/layer_10/attention/self/transpose_3_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/MatMul_1_grad/MatMul" op: "BatchMatMul" input: "gradients/bert/encoder/layer_10/attention/self/transpose_3_grad/transpose" input: "bert/encoder/layer_10/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "gradients/bert/encoder/layer_10/attention/self/MatMul_1_grad/MatMul_1" op: "BatchMatMul" input: "bert/encoder/layer_10/attention/self/dropout/mul" input: "gradients/bert/encoder/layer_10/attention/self/transpose_3_grad/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/attention/self/dropout/mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_10/attention/self/MatMul_1_grad/MatMul" input: "bert/encoder/layer_10/attention/self/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/dropout/mul_grad/Mul_1" op: "Mul" input: "gradients/bert/encoder/layer_10/attention/self/MatMul_1_grad/MatMul" input: "bert/encoder/layer_10/attention/self/dropout/div" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/transpose_2_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_10/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/transpose_2_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_10/attention/self/MatMul_1_grad/MatMul_1" input: "gradients/bert/encoder/layer_10/attention/self/transpose_2_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/dropout/div_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/dropout/div_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/dropout/div_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_10/attention/self/dropout/div_grad/Shape" input: "gradients/bert/encoder/layer_10/attention/self/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/dropout/div_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_10/attention/self/dropout/mul_grad/Mul" input: "bert/encoder/layer_10/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/dropout/div_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_10/attention/self/dropout/div_grad/RealDiv" input: "gradients/bert/encoder/layer_10/attention/self/dropout/div_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/attention/self/dropout/div_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/self/dropout/div_grad/Sum" input: "gradients/bert/encoder/layer_10/attention/self/dropout/div_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/dropout/div_grad/Neg" op: "Neg" input: "bert/encoder/layer_10/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/dropout/div_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_10/attention/self/dropout/div_grad/Neg" input: "bert/encoder/layer_10/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/dropout/div_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_10/attention/self/dropout/div_grad/RealDiv_1" input: "bert/encoder/layer_10/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/dropout/div_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_10/attention/self/dropout/mul_grad/Mul" input: "gradients/bert/encoder/layer_10/attention/self/dropout/div_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/dropout/div_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_10/attention/self/dropout/div_grad/mul" input: "gradients/bert/encoder/layer_10/attention/self/dropout/div_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/attention/self/dropout/div_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/self/dropout/div_grad/Sum_1" input: "gradients/bert/encoder/layer_10/attention/self/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Reshape_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Reshape_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/self/transpose_2_grad/transpose" input: "gradients/bert/encoder/layer_10/attention/self/Reshape_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Softmax_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_10/attention/self/dropout/div_grad/Reshape" input: "bert/encoder/layer_10/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Softmax_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: -1 } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Softmax_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_10/attention/self/Softmax_grad/mul" input: "gradients/bert/encoder/layer_10/attention/self/Softmax_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Softmax_grad/sub" op: "Sub" input: "gradients/bert/encoder/layer_10/attention/self/dropout/div_grad/Reshape" input: "gradients/bert/encoder/layer_10/attention/self/Softmax_grad/Sum" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Softmax_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_10/attention/self/Softmax_grad/sub" input: "bert/encoder/layer_10/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/value/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_10/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_10/attention/self/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\001\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_10/attention/self/add_grad/Shape" input: "gradients/bert/encoder/layer_10/attention/self/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_10/attention/self/Softmax_grad/mul_1" input: "gradients/bert/encoder/layer_10/attention/self/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/attention/self/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/self/add_grad/Sum" input: "gradients/bert/encoder/layer_10/attention/self/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_10/attention/self/Softmax_grad/mul_1" input: "gradients/bert/encoder/layer_10/attention/self/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/attention/self/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/self/add_grad/Sum_1" input: "gradients/bert/encoder/layer_10/attention/self/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/value/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_10/attention/self/Reshape_2_grad/Reshape" input: "bert/encoder/layer_10/attention/self/value/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_10/attention/self/value/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_9/output/LayerNorm/batchnorm/add_1" input: "gradients/bert/encoder/layer_10/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_10/attention/self/Mul_grad/Shape" input: "gradients/bert/encoder/layer_10/attention/self/Mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_10/attention/self/add_grad/Reshape" input: "bert/encoder/layer_10/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_10/attention/self/Mul_grad/Mul" input: "gradients/bert/encoder/layer_10/attention/self/Mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/self/Mul_grad/Sum" input: "gradients/bert/encoder/layer_10/attention/self/Mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_10/attention/self/MatMul" input: "gradients/bert/encoder/layer_10/attention/self/add_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_10/attention/self/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_10/attention/self/Mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/self/Mul_grad/Sum_1" input: "gradients/bert/encoder/layer_10/attention/self/Mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/MatMul_grad/MatMul" op: "BatchMatMul" input: "gradients/bert/encoder/layer_10/attention/self/Mul_grad/Reshape" input: "bert/encoder/layer_10/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/attention/self/MatMul_grad/MatMul_1" op: "BatchMatMul" input: "gradients/bert/encoder/layer_10/attention/self/Mul_grad/Reshape" input: "bert/encoder/layer_10/attention/self/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/attention/self/transpose_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_10/attention/self/transpose/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/transpose_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_10/attention/self/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_10/attention/self/transpose_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/transpose_1_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_10/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/transpose_1_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_10/attention/self/MatMul_grad/MatMul_1" input: "gradients/bert/encoder/layer_10/attention/self/transpose_1_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Reshape_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Reshape_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/self/transpose_grad/transpose" input: "gradients/bert/encoder/layer_10/attention/self/Reshape_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Reshape_1_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Reshape_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/self/transpose_1_grad/transpose" input: "gradients/bert/encoder/layer_10/attention/self/Reshape_1_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/query/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_10/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_10/attention/self/key/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_10/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_10/attention/self/query/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_10/attention/self/Reshape_grad/Reshape" input: "bert/encoder/layer_10/attention/self/query/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_10/attention/self/query/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_9/output/LayerNorm/batchnorm/add_1" input: "gradients/bert/encoder/layer_10/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/attention/self/key/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_10/attention/self/Reshape_1_grad/Reshape" input: "bert/encoder/layer_10/attention/self/key/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_10/attention/self/key/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_9/output/LayerNorm/batchnorm/add_1" input: "gradients/bert/encoder/layer_10/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_17" op: "AddN" input: "gradients/AddN_16" input: "gradients/bert/encoder/layer_10/attention/self/value/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_10/attention/self/query/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_10/attention/self/key/MatMul_grad/MatMul" attr { key: "N" value { i: 4 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_17" input: "bert/encoder/layer_9/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_17" input: "bert/encoder/layer_9/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Shape" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_17" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Sum_1" op: "Sum" input: "gradients/AddN_17" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Sum_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Neg" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Shape" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Reshape_1" input: "bert/encoder/layer_9/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_9/output/LayerNorm/moments/mean" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_18" op: "AddN" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Shape" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_18" input: "bert/encoder/layer_9/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_9/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/AddN_18" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_9/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/add_grad/Shape" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/add_grad/Sum_1" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/add" op: "Add" input: "bert/encoder/layer_9/output/LayerNorm/moments/variance/reduction_indices" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/add" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/range" op: "Range" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/range/start" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/Size" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/Shape_1" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/range" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/mod" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/DynamicStitch" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/add_grad/Reshape" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/Shape" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_9/output/add" input: "bert/encoder/layer_9/output/LayerNorm/moments/StopGradient" input: "^gradients/bert/encoder/layer_9/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/mul" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/add" op: "Add" input: "bert/encoder/layer_9/output/LayerNorm/moments/mean/reduction_indices" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/add" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/range" op: "Range" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/range/start" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/Size" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/Shape_1" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/range" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/mod" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/DynamicStitch" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_19" op: "AddN" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/Reshape" input: "gradients/bert/encoder/layer_9/output/LayerNorm/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/dropout/mul_grad/Mul" op: "Mul" input: "gradients/AddN_19" input: "bert/encoder/layer_9/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/dropout/mul_grad/Mul_1" op: "Mul" input: "gradients/AddN_19" input: "bert/encoder/layer_9/output/dropout/div" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/dropout/div_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/output/dropout/div_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_9/output/dropout/div_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_9/output/dropout/div_grad/Shape" input: "gradients/bert/encoder/layer_9/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/dropout/div_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_9/output/dropout/mul_grad/Mul" input: "bert/encoder/layer_9/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/dropout/div_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_9/output/dropout/div_grad/RealDiv" input: "gradients/bert/encoder/layer_9/output/dropout/div_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/output/dropout/div_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/output/dropout/div_grad/Sum" input: "gradients/bert/encoder/layer_9/output/dropout/div_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/dropout/div_grad/Neg" op: "Neg" input: "bert/encoder/layer_9/output/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/dropout/div_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_9/output/dropout/div_grad/Neg" input: "bert/encoder/layer_9/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/dropout/div_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_9/output/dropout/div_grad/RealDiv_1" input: "bert/encoder/layer_9/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/dropout/div_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_9/output/dropout/mul_grad/Mul" input: "gradients/bert/encoder/layer_9/output/dropout/div_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/dropout/div_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_9/output/dropout/div_grad/mul" input: "gradients/bert/encoder/layer_9/output/dropout/div_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/output/dropout/div_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_9/output/dropout/div_grad/Sum_1" input: "gradients/bert/encoder/layer_9/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_9/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_9/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_9/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_9/output/dropout/div_grad/Reshape" input: "bert/encoder/layer_9/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_9/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_9/intermediate/dense/mul_1" input: "gradients/bert/encoder/layer_9/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/mul_1_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_9/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_9/intermediate/dense/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/mul_1_grad/Mul_1" op: "Mul" input: "gradients/bert/encoder/layer_9/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_9/intermediate/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_9/intermediate/dense/mul_grad/Shape" input: "gradients/bert/encoder/layer_9/intermediate/dense/mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_9/intermediate/dense/mul_1_grad/Mul_1" input: "bert/encoder/layer_9/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_9/intermediate/dense/mul_grad/Mul" input: "gradients/bert/encoder/layer_9/intermediate/dense/mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/intermediate/dense/mul_grad/Sum" input: "gradients/bert/encoder/layer_9/intermediate/dense/mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_9/intermediate/dense/mul/x" input: "gradients/bert/encoder/layer_9/intermediate/dense/mul_1_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_9/intermediate/dense/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_9/intermediate/dense/mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_9/intermediate/dense/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_9/intermediate/dense/mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_9/intermediate/dense/add_grad/Shape" input: "gradients/bert/encoder/layer_9/intermediate/dense/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_9/intermediate/dense/mul_grad/Reshape_1" input: "gradients/bert/encoder/layer_9/intermediate/dense/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/intermediate/dense/add_grad/Sum" input: "gradients/bert/encoder/layer_9/intermediate/dense/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_9/intermediate/dense/mul_grad/Reshape_1" input: "gradients/bert/encoder/layer_9/intermediate/dense/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_9/intermediate/dense/add_grad/Sum_1" input: "gradients/bert/encoder/layer_9/intermediate/dense/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/Erf_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.128379225730896 } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/Erf_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_9/intermediate/dense/add_grad/Reshape_1" input: "gradients/bert/encoder/layer_9/intermediate/dense/Erf_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/Erf_grad/Square" op: "Square" input: "bert/encoder/layer_9/intermediate/dense/truediv" input: "^gradients/bert/encoder/layer_9/intermediate/dense/add_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/Erf_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_9/intermediate/dense/Erf_grad/Square" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/Erf_grad/Exp" op: "Exp" input: "gradients/bert/encoder/layer_9/intermediate/dense/Erf_grad/Neg" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/Erf_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_9/intermediate/dense/Erf_grad/mul" input: "gradients/bert/encoder/layer_9/intermediate/dense/Erf_grad/Exp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/truediv_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/truediv_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/truediv_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_9/intermediate/dense/truediv_grad/Shape" input: "gradients/bert/encoder/layer_9/intermediate/dense/truediv_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/truediv_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_9/intermediate/dense/Erf_grad/mul_1" input: "bert/encoder/layer_9/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/truediv_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_9/intermediate/dense/truediv_grad/RealDiv" input: "gradients/bert/encoder/layer_9/intermediate/dense/truediv_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/truediv_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/intermediate/dense/truediv_grad/Sum" input: "gradients/bert/encoder/layer_9/intermediate/dense/truediv_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/truediv_grad/Neg" op: "Neg" input: "bert/encoder/layer_9/intermediate/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/truediv_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_9/intermediate/dense/truediv_grad/Neg" input: "bert/encoder/layer_9/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/truediv_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_9/intermediate/dense/truediv_grad/RealDiv_1" input: "bert/encoder/layer_9/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/truediv_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_9/intermediate/dense/Erf_grad/mul_1" input: "gradients/bert/encoder/layer_9/intermediate/dense/truediv_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/truediv_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_9/intermediate/dense/truediv_grad/mul" input: "gradients/bert/encoder/layer_9/intermediate/dense/truediv_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/truediv_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_9/intermediate/dense/truediv_grad/Sum_1" input: "gradients/bert/encoder/layer_9/intermediate/dense/truediv_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/AddN_20" op: "AddN" input: "gradients/bert/encoder/layer_9/intermediate/dense/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_9/intermediate/dense/truediv_grad/Reshape" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/intermediate/dense/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/AddN_20" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/AddN_20" input: "bert/encoder/layer_9/intermediate/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_1" input: "gradients/AddN_20" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_21" op: "AddN" input: "gradients/AddN_19" input: "gradients/bert/encoder/layer_9/intermediate/dense/MatMul_grad/MatMul" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_21" input: "bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_21" input: "bert/encoder/layer_9/attention/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Shape" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_21" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Sum_1" op: "Sum" input: "gradients/AddN_21" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Sum_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Neg" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Reshape_1" input: "bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_9/attention/output/LayerNorm/moments/mean" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_22" op: "AddN" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Shape" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_22" input: "bert/encoder/layer_9/attention/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/AddN_22" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_grad/Shape" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_grad/Sum_1" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/add" op: "Add" input: "bert/encoder/layer_9/attention/output/LayerNorm/moments/variance/reduction_indices" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/add" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/range" op: "Range" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/range/start" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Size" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Shape_1" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/range" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/mod" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/DynamicStitch" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_grad/Reshape" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_9/attention/output/add" input: "bert/encoder/layer_9/attention/output/LayerNorm/moments/StopGradient" input: "^gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/mul" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/add" op: "Add" input: "bert/encoder/layer_9/attention/output/LayerNorm/moments/mean/reduction_indices" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/add" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/range" op: "Range" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/range/start" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Size" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Shape_1" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/range" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/mod" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/DynamicStitch" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_23" op: "AddN" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/dropout/mul_grad/Mul" op: "Mul" input: "gradients/AddN_23" input: "bert/encoder/layer_9/attention/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/dropout/mul_grad/Mul_1" op: "Mul" input: "gradients/AddN_23" input: "bert/encoder/layer_9/attention/output/dropout/div" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/dropout/div_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/dropout/div_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/dropout/div_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_9/attention/output/dropout/div_grad/Shape" input: "gradients/bert/encoder/layer_9/attention/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/dropout/div_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_9/attention/output/dropout/mul_grad/Mul" input: "bert/encoder/layer_9/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/dropout/div_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_9/attention/output/dropout/div_grad/RealDiv" input: "gradients/bert/encoder/layer_9/attention/output/dropout/div_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/attention/output/dropout/div_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/output/dropout/div_grad/Sum" input: "gradients/bert/encoder/layer_9/attention/output/dropout/div_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/dropout/div_grad/Neg" op: "Neg" input: "bert/encoder/layer_9/attention/output/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/dropout/div_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_9/attention/output/dropout/div_grad/Neg" input: "bert/encoder/layer_9/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/dropout/div_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_9/attention/output/dropout/div_grad/RealDiv_1" input: "bert/encoder/layer_9/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/dropout/div_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_9/attention/output/dropout/mul_grad/Mul" input: "gradients/bert/encoder/layer_9/attention/output/dropout/div_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/dropout/div_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_9/attention/output/dropout/div_grad/mul" input: "gradients/bert/encoder/layer_9/attention/output/dropout/div_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/attention/output/dropout/div_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/output/dropout/div_grad/Sum_1" input: "gradients/bert/encoder/layer_9/attention/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_9/attention/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_9/attention/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_9/attention/output/dropout/div_grad/Reshape" input: "bert/encoder/layer_9/attention/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_9/attention/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_9/attention/self/Reshape_3" input: "gradients/bert/encoder/layer_9/attention/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Reshape_3_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Reshape_3_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/output/dense/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_9/attention/self/Reshape_3_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/transpose_3_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_9/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/transpose_3_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_9/attention/self/Reshape_3_grad/Reshape" input: "gradients/bert/encoder/layer_9/attention/self/transpose_3_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/MatMul_1_grad/MatMul" op: "BatchMatMul" input: "gradients/bert/encoder/layer_9/attention/self/transpose_3_grad/transpose" input: "bert/encoder/layer_9/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "gradients/bert/encoder/layer_9/attention/self/MatMul_1_grad/MatMul_1" op: "BatchMatMul" input: "bert/encoder/layer_9/attention/self/dropout/mul" input: "gradients/bert/encoder/layer_9/attention/self/transpose_3_grad/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/attention/self/dropout/mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_9/attention/self/MatMul_1_grad/MatMul" input: "bert/encoder/layer_9/attention/self/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/dropout/mul_grad/Mul_1" op: "Mul" input: "gradients/bert/encoder/layer_9/attention/self/MatMul_1_grad/MatMul" input: "bert/encoder/layer_9/attention/self/dropout/div" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/transpose_2_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_9/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/transpose_2_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_9/attention/self/MatMul_1_grad/MatMul_1" input: "gradients/bert/encoder/layer_9/attention/self/transpose_2_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/dropout/div_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/dropout/div_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/dropout/div_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_9/attention/self/dropout/div_grad/Shape" input: "gradients/bert/encoder/layer_9/attention/self/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/dropout/div_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_9/attention/self/dropout/mul_grad/Mul" input: "bert/encoder/layer_9/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/dropout/div_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_9/attention/self/dropout/div_grad/RealDiv" input: "gradients/bert/encoder/layer_9/attention/self/dropout/div_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/attention/self/dropout/div_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/self/dropout/div_grad/Sum" input: "gradients/bert/encoder/layer_9/attention/self/dropout/div_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/dropout/div_grad/Neg" op: "Neg" input: "bert/encoder/layer_9/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/dropout/div_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_9/attention/self/dropout/div_grad/Neg" input: "bert/encoder/layer_9/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/dropout/div_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_9/attention/self/dropout/div_grad/RealDiv_1" input: "bert/encoder/layer_9/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/dropout/div_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_9/attention/self/dropout/mul_grad/Mul" input: "gradients/bert/encoder/layer_9/attention/self/dropout/div_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/dropout/div_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_9/attention/self/dropout/div_grad/mul" input: "gradients/bert/encoder/layer_9/attention/self/dropout/div_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/attention/self/dropout/div_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/self/dropout/div_grad/Sum_1" input: "gradients/bert/encoder/layer_9/attention/self/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Reshape_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Reshape_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/self/transpose_2_grad/transpose" input: "gradients/bert/encoder/layer_9/attention/self/Reshape_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Softmax_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_9/attention/self/dropout/div_grad/Reshape" input: "bert/encoder/layer_9/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Softmax_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: -1 } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Softmax_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_9/attention/self/Softmax_grad/mul" input: "gradients/bert/encoder/layer_9/attention/self/Softmax_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Softmax_grad/sub" op: "Sub" input: "gradients/bert/encoder/layer_9/attention/self/dropout/div_grad/Reshape" input: "gradients/bert/encoder/layer_9/attention/self/Softmax_grad/Sum" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Softmax_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_9/attention/self/Softmax_grad/sub" input: "bert/encoder/layer_9/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/value/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_9/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_9/attention/self/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\001\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_9/attention/self/add_grad/Shape" input: "gradients/bert/encoder/layer_9/attention/self/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_9/attention/self/Softmax_grad/mul_1" input: "gradients/bert/encoder/layer_9/attention/self/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/attention/self/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/self/add_grad/Sum" input: "gradients/bert/encoder/layer_9/attention/self/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_9/attention/self/Softmax_grad/mul_1" input: "gradients/bert/encoder/layer_9/attention/self/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/attention/self/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/self/add_grad/Sum_1" input: "gradients/bert/encoder/layer_9/attention/self/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/value/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_9/attention/self/Reshape_2_grad/Reshape" input: "bert/encoder/layer_9/attention/self/value/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_9/attention/self/value/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_8/output/LayerNorm/batchnorm/add_1" input: "gradients/bert/encoder/layer_9/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_9/attention/self/Mul_grad/Shape" input: "gradients/bert/encoder/layer_9/attention/self/Mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_9/attention/self/add_grad/Reshape" input: "bert/encoder/layer_9/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_9/attention/self/Mul_grad/Mul" input: "gradients/bert/encoder/layer_9/attention/self/Mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/self/Mul_grad/Sum" input: "gradients/bert/encoder/layer_9/attention/self/Mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_9/attention/self/MatMul" input: "gradients/bert/encoder/layer_9/attention/self/add_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_9/attention/self/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_9/attention/self/Mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/self/Mul_grad/Sum_1" input: "gradients/bert/encoder/layer_9/attention/self/Mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/MatMul_grad/MatMul" op: "BatchMatMul" input: "gradients/bert/encoder/layer_9/attention/self/Mul_grad/Reshape" input: "bert/encoder/layer_9/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/attention/self/MatMul_grad/MatMul_1" op: "BatchMatMul" input: "gradients/bert/encoder/layer_9/attention/self/Mul_grad/Reshape" input: "bert/encoder/layer_9/attention/self/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/attention/self/transpose_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_9/attention/self/transpose/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/transpose_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_9/attention/self/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_9/attention/self/transpose_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/transpose_1_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_9/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/transpose_1_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_9/attention/self/MatMul_grad/MatMul_1" input: "gradients/bert/encoder/layer_9/attention/self/transpose_1_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Reshape_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Reshape_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/self/transpose_grad/transpose" input: "gradients/bert/encoder/layer_9/attention/self/Reshape_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Reshape_1_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Reshape_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/self/transpose_1_grad/transpose" input: "gradients/bert/encoder/layer_9/attention/self/Reshape_1_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/query/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_9/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_9/attention/self/key/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_9/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_9/attention/self/query/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_9/attention/self/Reshape_grad/Reshape" input: "bert/encoder/layer_9/attention/self/query/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_9/attention/self/query/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_8/output/LayerNorm/batchnorm/add_1" input: "gradients/bert/encoder/layer_9/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/attention/self/key/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_9/attention/self/Reshape_1_grad/Reshape" input: "bert/encoder/layer_9/attention/self/key/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_9/attention/self/key/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_8/output/LayerNorm/batchnorm/add_1" input: "gradients/bert/encoder/layer_9/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_24" op: "AddN" input: "gradients/AddN_23" input: "gradients/bert/encoder/layer_9/attention/self/value/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_9/attention/self/query/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_9/attention/self/key/MatMul_grad/MatMul" attr { key: "N" value { i: 4 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_24" input: "bert/encoder/layer_8/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_24" input: "bert/encoder/layer_8/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Shape" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_24" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Sum_1" op: "Sum" input: "gradients/AddN_24" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Sum_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Neg" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Shape" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Reshape_1" input: "bert/encoder/layer_8/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_8/output/LayerNorm/moments/mean" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_25" op: "AddN" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Shape" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_25" input: "bert/encoder/layer_8/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_8/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/AddN_25" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_8/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/add_grad/Shape" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/add_grad/Sum_1" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/add" op: "Add" input: "bert/encoder/layer_8/output/LayerNorm/moments/variance/reduction_indices" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/add" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/range" op: "Range" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/range/start" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/Size" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/Shape_1" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/range" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/mod" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/DynamicStitch" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/add_grad/Reshape" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/Shape" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_8/output/add" input: "bert/encoder/layer_8/output/LayerNorm/moments/StopGradient" input: "^gradients/bert/encoder/layer_8/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/mul" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/add" op: "Add" input: "bert/encoder/layer_8/output/LayerNorm/moments/mean/reduction_indices" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/add" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/range" op: "Range" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/range/start" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/Size" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/Shape_1" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/range" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/mod" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/DynamicStitch" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_26" op: "AddN" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/Reshape" input: "gradients/bert/encoder/layer_8/output/LayerNorm/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/dropout/mul_grad/Mul" op: "Mul" input: "gradients/AddN_26" input: "bert/encoder/layer_8/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/dropout/mul_grad/Mul_1" op: "Mul" input: "gradients/AddN_26" input: "bert/encoder/layer_8/output/dropout/div" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/dropout/div_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/output/dropout/div_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_8/output/dropout/div_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_8/output/dropout/div_grad/Shape" input: "gradients/bert/encoder/layer_8/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/dropout/div_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_8/output/dropout/mul_grad/Mul" input: "bert/encoder/layer_8/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/dropout/div_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_8/output/dropout/div_grad/RealDiv" input: "gradients/bert/encoder/layer_8/output/dropout/div_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/output/dropout/div_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/output/dropout/div_grad/Sum" input: "gradients/bert/encoder/layer_8/output/dropout/div_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/dropout/div_grad/Neg" op: "Neg" input: "bert/encoder/layer_8/output/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/dropout/div_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_8/output/dropout/div_grad/Neg" input: "bert/encoder/layer_8/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/dropout/div_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_8/output/dropout/div_grad/RealDiv_1" input: "bert/encoder/layer_8/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/dropout/div_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_8/output/dropout/mul_grad/Mul" input: "gradients/bert/encoder/layer_8/output/dropout/div_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/dropout/div_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_8/output/dropout/div_grad/mul" input: "gradients/bert/encoder/layer_8/output/dropout/div_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/output/dropout/div_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_8/output/dropout/div_grad/Sum_1" input: "gradients/bert/encoder/layer_8/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_8/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_8/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_8/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_8/output/dropout/div_grad/Reshape" input: "bert/encoder/layer_8/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_8/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_8/intermediate/dense/mul_1" input: "gradients/bert/encoder/layer_8/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/mul_1_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_8/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_8/intermediate/dense/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/mul_1_grad/Mul_1" op: "Mul" input: "gradients/bert/encoder/layer_8/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_8/intermediate/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_8/intermediate/dense/mul_grad/Shape" input: "gradients/bert/encoder/layer_8/intermediate/dense/mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_8/intermediate/dense/mul_1_grad/Mul_1" input: "bert/encoder/layer_8/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_8/intermediate/dense/mul_grad/Mul" input: "gradients/bert/encoder/layer_8/intermediate/dense/mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/intermediate/dense/mul_grad/Sum" input: "gradients/bert/encoder/layer_8/intermediate/dense/mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_8/intermediate/dense/mul/x" input: "gradients/bert/encoder/layer_8/intermediate/dense/mul_1_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_8/intermediate/dense/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_8/intermediate/dense/mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_8/intermediate/dense/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_8/intermediate/dense/mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_8/intermediate/dense/add_grad/Shape" input: "gradients/bert/encoder/layer_8/intermediate/dense/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_8/intermediate/dense/mul_grad/Reshape_1" input: "gradients/bert/encoder/layer_8/intermediate/dense/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/intermediate/dense/add_grad/Sum" input: "gradients/bert/encoder/layer_8/intermediate/dense/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_8/intermediate/dense/mul_grad/Reshape_1" input: "gradients/bert/encoder/layer_8/intermediate/dense/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_8/intermediate/dense/add_grad/Sum_1" input: "gradients/bert/encoder/layer_8/intermediate/dense/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/Erf_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.128379225730896 } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/Erf_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_8/intermediate/dense/add_grad/Reshape_1" input: "gradients/bert/encoder/layer_8/intermediate/dense/Erf_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/Erf_grad/Square" op: "Square" input: "bert/encoder/layer_8/intermediate/dense/truediv" input: "^gradients/bert/encoder/layer_8/intermediate/dense/add_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/Erf_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_8/intermediate/dense/Erf_grad/Square" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/Erf_grad/Exp" op: "Exp" input: "gradients/bert/encoder/layer_8/intermediate/dense/Erf_grad/Neg" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/Erf_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_8/intermediate/dense/Erf_grad/mul" input: "gradients/bert/encoder/layer_8/intermediate/dense/Erf_grad/Exp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/truediv_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/truediv_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/truediv_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_8/intermediate/dense/truediv_grad/Shape" input: "gradients/bert/encoder/layer_8/intermediate/dense/truediv_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/truediv_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_8/intermediate/dense/Erf_grad/mul_1" input: "bert/encoder/layer_8/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/truediv_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_8/intermediate/dense/truediv_grad/RealDiv" input: "gradients/bert/encoder/layer_8/intermediate/dense/truediv_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/truediv_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/intermediate/dense/truediv_grad/Sum" input: "gradients/bert/encoder/layer_8/intermediate/dense/truediv_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/truediv_grad/Neg" op: "Neg" input: "bert/encoder/layer_8/intermediate/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/truediv_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_8/intermediate/dense/truediv_grad/Neg" input: "bert/encoder/layer_8/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/truediv_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_8/intermediate/dense/truediv_grad/RealDiv_1" input: "bert/encoder/layer_8/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/truediv_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_8/intermediate/dense/Erf_grad/mul_1" input: "gradients/bert/encoder/layer_8/intermediate/dense/truediv_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/truediv_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_8/intermediate/dense/truediv_grad/mul" input: "gradients/bert/encoder/layer_8/intermediate/dense/truediv_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/truediv_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_8/intermediate/dense/truediv_grad/Sum_1" input: "gradients/bert/encoder/layer_8/intermediate/dense/truediv_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/AddN_27" op: "AddN" input: "gradients/bert/encoder/layer_8/intermediate/dense/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_8/intermediate/dense/truediv_grad/Reshape" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/intermediate/dense/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/AddN_27" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/AddN_27" input: "bert/encoder/layer_8/intermediate/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_1" input: "gradients/AddN_27" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_28" op: "AddN" input: "gradients/AddN_26" input: "gradients/bert/encoder/layer_8/intermediate/dense/MatMul_grad/MatMul" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_28" input: "bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_28" input: "bert/encoder/layer_8/attention/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Shape" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_28" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Sum_1" op: "Sum" input: "gradients/AddN_28" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Sum_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Neg" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Reshape_1" input: "bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_8/attention/output/LayerNorm/moments/mean" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_29" op: "AddN" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Shape" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_29" input: "bert/encoder/layer_8/attention/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/AddN_29" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_grad/Shape" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_grad/Sum_1" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/add" op: "Add" input: "bert/encoder/layer_8/attention/output/LayerNorm/moments/variance/reduction_indices" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/add" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/range" op: "Range" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/range/start" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Size" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Shape_1" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/range" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/mod" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/DynamicStitch" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_grad/Reshape" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_8/attention/output/add" input: "bert/encoder/layer_8/attention/output/LayerNorm/moments/StopGradient" input: "^gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/mul" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/add" op: "Add" input: "bert/encoder/layer_8/attention/output/LayerNorm/moments/mean/reduction_indices" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/add" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/range" op: "Range" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/range/start" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Size" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Shape_1" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/range" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/mod" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/DynamicStitch" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_30" op: "AddN" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/dropout/mul_grad/Mul" op: "Mul" input: "gradients/AddN_30" input: "bert/encoder/layer_8/attention/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/dropout/mul_grad/Mul_1" op: "Mul" input: "gradients/AddN_30" input: "bert/encoder/layer_8/attention/output/dropout/div" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/dropout/div_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/dropout/div_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/dropout/div_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_8/attention/output/dropout/div_grad/Shape" input: "gradients/bert/encoder/layer_8/attention/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/dropout/div_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_8/attention/output/dropout/mul_grad/Mul" input: "bert/encoder/layer_8/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/dropout/div_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_8/attention/output/dropout/div_grad/RealDiv" input: "gradients/bert/encoder/layer_8/attention/output/dropout/div_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/attention/output/dropout/div_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/output/dropout/div_grad/Sum" input: "gradients/bert/encoder/layer_8/attention/output/dropout/div_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/dropout/div_grad/Neg" op: "Neg" input: "bert/encoder/layer_8/attention/output/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/dropout/div_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_8/attention/output/dropout/div_grad/Neg" input: "bert/encoder/layer_8/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/dropout/div_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_8/attention/output/dropout/div_grad/RealDiv_1" input: "bert/encoder/layer_8/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/dropout/div_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_8/attention/output/dropout/mul_grad/Mul" input: "gradients/bert/encoder/layer_8/attention/output/dropout/div_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/dropout/div_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_8/attention/output/dropout/div_grad/mul" input: "gradients/bert/encoder/layer_8/attention/output/dropout/div_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/attention/output/dropout/div_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/output/dropout/div_grad/Sum_1" input: "gradients/bert/encoder/layer_8/attention/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_8/attention/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_8/attention/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_8/attention/output/dropout/div_grad/Reshape" input: "bert/encoder/layer_8/attention/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_8/attention/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_8/attention/self/Reshape_3" input: "gradients/bert/encoder/layer_8/attention/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Reshape_3_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Reshape_3_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/output/dense/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_8/attention/self/Reshape_3_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/transpose_3_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_8/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/transpose_3_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_8/attention/self/Reshape_3_grad/Reshape" input: "gradients/bert/encoder/layer_8/attention/self/transpose_3_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/MatMul_1_grad/MatMul" op: "BatchMatMul" input: "gradients/bert/encoder/layer_8/attention/self/transpose_3_grad/transpose" input: "bert/encoder/layer_8/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "gradients/bert/encoder/layer_8/attention/self/MatMul_1_grad/MatMul_1" op: "BatchMatMul" input: "bert/encoder/layer_8/attention/self/dropout/mul" input: "gradients/bert/encoder/layer_8/attention/self/transpose_3_grad/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/attention/self/dropout/mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_8/attention/self/MatMul_1_grad/MatMul" input: "bert/encoder/layer_8/attention/self/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/dropout/mul_grad/Mul_1" op: "Mul" input: "gradients/bert/encoder/layer_8/attention/self/MatMul_1_grad/MatMul" input: "bert/encoder/layer_8/attention/self/dropout/div" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/transpose_2_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_8/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/transpose_2_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_8/attention/self/MatMul_1_grad/MatMul_1" input: "gradients/bert/encoder/layer_8/attention/self/transpose_2_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/dropout/div_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/dropout/div_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/dropout/div_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_8/attention/self/dropout/div_grad/Shape" input: "gradients/bert/encoder/layer_8/attention/self/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/dropout/div_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_8/attention/self/dropout/mul_grad/Mul" input: "bert/encoder/layer_8/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/dropout/div_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_8/attention/self/dropout/div_grad/RealDiv" input: "gradients/bert/encoder/layer_8/attention/self/dropout/div_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/attention/self/dropout/div_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/self/dropout/div_grad/Sum" input: "gradients/bert/encoder/layer_8/attention/self/dropout/div_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/dropout/div_grad/Neg" op: "Neg" input: "bert/encoder/layer_8/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/dropout/div_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_8/attention/self/dropout/div_grad/Neg" input: "bert/encoder/layer_8/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/dropout/div_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_8/attention/self/dropout/div_grad/RealDiv_1" input: "bert/encoder/layer_8/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/dropout/div_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_8/attention/self/dropout/mul_grad/Mul" input: "gradients/bert/encoder/layer_8/attention/self/dropout/div_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/dropout/div_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_8/attention/self/dropout/div_grad/mul" input: "gradients/bert/encoder/layer_8/attention/self/dropout/div_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/attention/self/dropout/div_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/self/dropout/div_grad/Sum_1" input: "gradients/bert/encoder/layer_8/attention/self/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Reshape_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Reshape_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/self/transpose_2_grad/transpose" input: "gradients/bert/encoder/layer_8/attention/self/Reshape_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Softmax_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_8/attention/self/dropout/div_grad/Reshape" input: "bert/encoder/layer_8/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Softmax_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: -1 } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Softmax_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_8/attention/self/Softmax_grad/mul" input: "gradients/bert/encoder/layer_8/attention/self/Softmax_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Softmax_grad/sub" op: "Sub" input: "gradients/bert/encoder/layer_8/attention/self/dropout/div_grad/Reshape" input: "gradients/bert/encoder/layer_8/attention/self/Softmax_grad/Sum" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Softmax_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_8/attention/self/Softmax_grad/sub" input: "bert/encoder/layer_8/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/value/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_8/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_8/attention/self/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\001\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_8/attention/self/add_grad/Shape" input: "gradients/bert/encoder/layer_8/attention/self/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_8/attention/self/Softmax_grad/mul_1" input: "gradients/bert/encoder/layer_8/attention/self/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/attention/self/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/self/add_grad/Sum" input: "gradients/bert/encoder/layer_8/attention/self/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_8/attention/self/Softmax_grad/mul_1" input: "gradients/bert/encoder/layer_8/attention/self/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/attention/self/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/self/add_grad/Sum_1" input: "gradients/bert/encoder/layer_8/attention/self/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/value/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_8/attention/self/Reshape_2_grad/Reshape" input: "bert/encoder/layer_8/attention/self/value/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_8/attention/self/value/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_7/output/LayerNorm/batchnorm/add_1" input: "gradients/bert/encoder/layer_8/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_8/attention/self/Mul_grad/Shape" input: "gradients/bert/encoder/layer_8/attention/self/Mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_8/attention/self/add_grad/Reshape" input: "bert/encoder/layer_8/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_8/attention/self/Mul_grad/Mul" input: "gradients/bert/encoder/layer_8/attention/self/Mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/self/Mul_grad/Sum" input: "gradients/bert/encoder/layer_8/attention/self/Mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_8/attention/self/MatMul" input: "gradients/bert/encoder/layer_8/attention/self/add_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_8/attention/self/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_8/attention/self/Mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/self/Mul_grad/Sum_1" input: "gradients/bert/encoder/layer_8/attention/self/Mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/MatMul_grad/MatMul" op: "BatchMatMul" input: "gradients/bert/encoder/layer_8/attention/self/Mul_grad/Reshape" input: "bert/encoder/layer_8/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/attention/self/MatMul_grad/MatMul_1" op: "BatchMatMul" input: "gradients/bert/encoder/layer_8/attention/self/Mul_grad/Reshape" input: "bert/encoder/layer_8/attention/self/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/attention/self/transpose_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_8/attention/self/transpose/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/transpose_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_8/attention/self/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_8/attention/self/transpose_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/transpose_1_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_8/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/transpose_1_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_8/attention/self/MatMul_grad/MatMul_1" input: "gradients/bert/encoder/layer_8/attention/self/transpose_1_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Reshape_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Reshape_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/self/transpose_grad/transpose" input: "gradients/bert/encoder/layer_8/attention/self/Reshape_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Reshape_1_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Reshape_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/self/transpose_1_grad/transpose" input: "gradients/bert/encoder/layer_8/attention/self/Reshape_1_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/query/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_8/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_8/attention/self/key/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_8/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_8/attention/self/query/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_8/attention/self/Reshape_grad/Reshape" input: "bert/encoder/layer_8/attention/self/query/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_8/attention/self/query/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_7/output/LayerNorm/batchnorm/add_1" input: "gradients/bert/encoder/layer_8/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/attention/self/key/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_8/attention/self/Reshape_1_grad/Reshape" input: "bert/encoder/layer_8/attention/self/key/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_8/attention/self/key/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_7/output/LayerNorm/batchnorm/add_1" input: "gradients/bert/encoder/layer_8/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_31" op: "AddN" input: "gradients/AddN_30" input: "gradients/bert/encoder/layer_8/attention/self/value/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_8/attention/self/query/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_8/attention/self/key/MatMul_grad/MatMul" attr { key: "N" value { i: 4 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_31" input: "bert/encoder/layer_7/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_31" input: "bert/encoder/layer_7/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Shape" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_31" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Sum_1" op: "Sum" input: "gradients/AddN_31" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Sum_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Neg" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Shape" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Reshape_1" input: "bert/encoder/layer_7/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_7/output/LayerNorm/moments/mean" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_32" op: "AddN" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Shape" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_32" input: "bert/encoder/layer_7/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_7/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/AddN_32" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_7/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/add_grad/Shape" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/add_grad/Sum_1" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/add" op: "Add" input: "bert/encoder/layer_7/output/LayerNorm/moments/variance/reduction_indices" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/add" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/range" op: "Range" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/range/start" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/Size" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/Shape_1" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/range" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/mod" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/DynamicStitch" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/add_grad/Reshape" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/Shape" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_7/output/add" input: "bert/encoder/layer_7/output/LayerNorm/moments/StopGradient" input: "^gradients/bert/encoder/layer_7/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/mul" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/add" op: "Add" input: "bert/encoder/layer_7/output/LayerNorm/moments/mean/reduction_indices" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/add" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/range" op: "Range" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/range/start" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/Size" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/Shape_1" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/range" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/mod" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/DynamicStitch" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_33" op: "AddN" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/Reshape" input: "gradients/bert/encoder/layer_7/output/LayerNorm/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/dropout/mul_grad/Mul" op: "Mul" input: "gradients/AddN_33" input: "bert/encoder/layer_7/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/dropout/mul_grad/Mul_1" op: "Mul" input: "gradients/AddN_33" input: "bert/encoder/layer_7/output/dropout/div" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/dropout/div_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/output/dropout/div_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_7/output/dropout/div_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_7/output/dropout/div_grad/Shape" input: "gradients/bert/encoder/layer_7/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/dropout/div_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_7/output/dropout/mul_grad/Mul" input: "bert/encoder/layer_7/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/dropout/div_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_7/output/dropout/div_grad/RealDiv" input: "gradients/bert/encoder/layer_7/output/dropout/div_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/output/dropout/div_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/output/dropout/div_grad/Sum" input: "gradients/bert/encoder/layer_7/output/dropout/div_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/dropout/div_grad/Neg" op: "Neg" input: "bert/encoder/layer_7/output/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/dropout/div_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_7/output/dropout/div_grad/Neg" input: "bert/encoder/layer_7/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/dropout/div_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_7/output/dropout/div_grad/RealDiv_1" input: "bert/encoder/layer_7/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/dropout/div_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_7/output/dropout/mul_grad/Mul" input: "gradients/bert/encoder/layer_7/output/dropout/div_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/dropout/div_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_7/output/dropout/div_grad/mul" input: "gradients/bert/encoder/layer_7/output/dropout/div_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/output/dropout/div_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_7/output/dropout/div_grad/Sum_1" input: "gradients/bert/encoder/layer_7/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_7/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_7/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_7/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_7/output/dropout/div_grad/Reshape" input: "bert/encoder/layer_7/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_7/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_7/intermediate/dense/mul_1" input: "gradients/bert/encoder/layer_7/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/mul_1_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_7/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_7/intermediate/dense/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/mul_1_grad/Mul_1" op: "Mul" input: "gradients/bert/encoder/layer_7/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_7/intermediate/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_7/intermediate/dense/mul_grad/Shape" input: "gradients/bert/encoder/layer_7/intermediate/dense/mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_7/intermediate/dense/mul_1_grad/Mul_1" input: "bert/encoder/layer_7/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_7/intermediate/dense/mul_grad/Mul" input: "gradients/bert/encoder/layer_7/intermediate/dense/mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/intermediate/dense/mul_grad/Sum" input: "gradients/bert/encoder/layer_7/intermediate/dense/mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_7/intermediate/dense/mul/x" input: "gradients/bert/encoder/layer_7/intermediate/dense/mul_1_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_7/intermediate/dense/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_7/intermediate/dense/mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_7/intermediate/dense/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_7/intermediate/dense/mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_7/intermediate/dense/add_grad/Shape" input: "gradients/bert/encoder/layer_7/intermediate/dense/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_7/intermediate/dense/mul_grad/Reshape_1" input: "gradients/bert/encoder/layer_7/intermediate/dense/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/intermediate/dense/add_grad/Sum" input: "gradients/bert/encoder/layer_7/intermediate/dense/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_7/intermediate/dense/mul_grad/Reshape_1" input: "gradients/bert/encoder/layer_7/intermediate/dense/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_7/intermediate/dense/add_grad/Sum_1" input: "gradients/bert/encoder/layer_7/intermediate/dense/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/Erf_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.128379225730896 } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/Erf_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_7/intermediate/dense/add_grad/Reshape_1" input: "gradients/bert/encoder/layer_7/intermediate/dense/Erf_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/Erf_grad/Square" op: "Square" input: "bert/encoder/layer_7/intermediate/dense/truediv" input: "^gradients/bert/encoder/layer_7/intermediate/dense/add_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/Erf_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_7/intermediate/dense/Erf_grad/Square" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/Erf_grad/Exp" op: "Exp" input: "gradients/bert/encoder/layer_7/intermediate/dense/Erf_grad/Neg" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/Erf_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_7/intermediate/dense/Erf_grad/mul" input: "gradients/bert/encoder/layer_7/intermediate/dense/Erf_grad/Exp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/truediv_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/truediv_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/truediv_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_7/intermediate/dense/truediv_grad/Shape" input: "gradients/bert/encoder/layer_7/intermediate/dense/truediv_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/truediv_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_7/intermediate/dense/Erf_grad/mul_1" input: "bert/encoder/layer_7/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/truediv_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_7/intermediate/dense/truediv_grad/RealDiv" input: "gradients/bert/encoder/layer_7/intermediate/dense/truediv_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/truediv_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/intermediate/dense/truediv_grad/Sum" input: "gradients/bert/encoder/layer_7/intermediate/dense/truediv_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/truediv_grad/Neg" op: "Neg" input: "bert/encoder/layer_7/intermediate/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/truediv_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_7/intermediate/dense/truediv_grad/Neg" input: "bert/encoder/layer_7/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/truediv_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_7/intermediate/dense/truediv_grad/RealDiv_1" input: "bert/encoder/layer_7/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/truediv_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_7/intermediate/dense/Erf_grad/mul_1" input: "gradients/bert/encoder/layer_7/intermediate/dense/truediv_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/truediv_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_7/intermediate/dense/truediv_grad/mul" input: "gradients/bert/encoder/layer_7/intermediate/dense/truediv_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/truediv_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_7/intermediate/dense/truediv_grad/Sum_1" input: "gradients/bert/encoder/layer_7/intermediate/dense/truediv_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/AddN_34" op: "AddN" input: "gradients/bert/encoder/layer_7/intermediate/dense/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_7/intermediate/dense/truediv_grad/Reshape" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/intermediate/dense/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/AddN_34" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/AddN_34" input: "bert/encoder/layer_7/intermediate/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_1" input: "gradients/AddN_34" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_35" op: "AddN" input: "gradients/AddN_33" input: "gradients/bert/encoder/layer_7/intermediate/dense/MatMul_grad/MatMul" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_35" input: "bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_35" input: "bert/encoder/layer_7/attention/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Shape" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_35" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Sum_1" op: "Sum" input: "gradients/AddN_35" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Sum_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Neg" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Reshape_1" input: "bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_7/attention/output/LayerNorm/moments/mean" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_36" op: "AddN" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Shape" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_36" input: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/AddN_36" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_grad/Shape" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_grad/Sum_1" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/add" op: "Add" input: "bert/encoder/layer_7/attention/output/LayerNorm/moments/variance/reduction_indices" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/add" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/range" op: "Range" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/range/start" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Size" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Shape_1" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/range" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/mod" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/DynamicStitch" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_grad/Reshape" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_7/attention/output/add" input: "bert/encoder/layer_7/attention/output/LayerNorm/moments/StopGradient" input: "^gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/mul" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/add" op: "Add" input: "bert/encoder/layer_7/attention/output/LayerNorm/moments/mean/reduction_indices" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/add" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/range" op: "Range" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/range/start" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Size" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Shape_1" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/range" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/mod" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/DynamicStitch" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_37" op: "AddN" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/dropout/mul_grad/Mul" op: "Mul" input: "gradients/AddN_37" input: "bert/encoder/layer_7/attention/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/dropout/mul_grad/Mul_1" op: "Mul" input: "gradients/AddN_37" input: "bert/encoder/layer_7/attention/output/dropout/div" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/dropout/div_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/dropout/div_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/dropout/div_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_7/attention/output/dropout/div_grad/Shape" input: "gradients/bert/encoder/layer_7/attention/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/dropout/div_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_7/attention/output/dropout/mul_grad/Mul" input: "bert/encoder/layer_7/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/dropout/div_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_7/attention/output/dropout/div_grad/RealDiv" input: "gradients/bert/encoder/layer_7/attention/output/dropout/div_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/attention/output/dropout/div_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/output/dropout/div_grad/Sum" input: "gradients/bert/encoder/layer_7/attention/output/dropout/div_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/dropout/div_grad/Neg" op: "Neg" input: "bert/encoder/layer_7/attention/output/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/dropout/div_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_7/attention/output/dropout/div_grad/Neg" input: "bert/encoder/layer_7/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/dropout/div_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_7/attention/output/dropout/div_grad/RealDiv_1" input: "bert/encoder/layer_7/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/dropout/div_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_7/attention/output/dropout/mul_grad/Mul" input: "gradients/bert/encoder/layer_7/attention/output/dropout/div_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/dropout/div_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_7/attention/output/dropout/div_grad/mul" input: "gradients/bert/encoder/layer_7/attention/output/dropout/div_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/attention/output/dropout/div_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/output/dropout/div_grad/Sum_1" input: "gradients/bert/encoder/layer_7/attention/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_7/attention/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_7/attention/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_7/attention/output/dropout/div_grad/Reshape" input: "bert/encoder/layer_7/attention/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_7/attention/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_7/attention/self/Reshape_3" input: "gradients/bert/encoder/layer_7/attention/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Reshape_3_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Reshape_3_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/output/dense/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_7/attention/self/Reshape_3_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/transpose_3_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_7/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/transpose_3_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_7/attention/self/Reshape_3_grad/Reshape" input: "gradients/bert/encoder/layer_7/attention/self/transpose_3_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/MatMul_1_grad/MatMul" op: "BatchMatMul" input: "gradients/bert/encoder/layer_7/attention/self/transpose_3_grad/transpose" input: "bert/encoder/layer_7/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "gradients/bert/encoder/layer_7/attention/self/MatMul_1_grad/MatMul_1" op: "BatchMatMul" input: "bert/encoder/layer_7/attention/self/dropout/mul" input: "gradients/bert/encoder/layer_7/attention/self/transpose_3_grad/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/attention/self/dropout/mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_7/attention/self/MatMul_1_grad/MatMul" input: "bert/encoder/layer_7/attention/self/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/dropout/mul_grad/Mul_1" op: "Mul" input: "gradients/bert/encoder/layer_7/attention/self/MatMul_1_grad/MatMul" input: "bert/encoder/layer_7/attention/self/dropout/div" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/transpose_2_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_7/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/transpose_2_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_7/attention/self/MatMul_1_grad/MatMul_1" input: "gradients/bert/encoder/layer_7/attention/self/transpose_2_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/dropout/div_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/dropout/div_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/dropout/div_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_7/attention/self/dropout/div_grad/Shape" input: "gradients/bert/encoder/layer_7/attention/self/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/dropout/div_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_7/attention/self/dropout/mul_grad/Mul" input: "bert/encoder/layer_7/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/dropout/div_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_7/attention/self/dropout/div_grad/RealDiv" input: "gradients/bert/encoder/layer_7/attention/self/dropout/div_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/attention/self/dropout/div_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/self/dropout/div_grad/Sum" input: "gradients/bert/encoder/layer_7/attention/self/dropout/div_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/dropout/div_grad/Neg" op: "Neg" input: "bert/encoder/layer_7/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/dropout/div_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_7/attention/self/dropout/div_grad/Neg" input: "bert/encoder/layer_7/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/dropout/div_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_7/attention/self/dropout/div_grad/RealDiv_1" input: "bert/encoder/layer_7/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/dropout/div_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_7/attention/self/dropout/mul_grad/Mul" input: "gradients/bert/encoder/layer_7/attention/self/dropout/div_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/dropout/div_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_7/attention/self/dropout/div_grad/mul" input: "gradients/bert/encoder/layer_7/attention/self/dropout/div_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/attention/self/dropout/div_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/self/dropout/div_grad/Sum_1" input: "gradients/bert/encoder/layer_7/attention/self/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Reshape_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Reshape_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/self/transpose_2_grad/transpose" input: "gradients/bert/encoder/layer_7/attention/self/Reshape_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Softmax_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_7/attention/self/dropout/div_grad/Reshape" input: "bert/encoder/layer_7/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Softmax_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: -1 } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Softmax_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_7/attention/self/Softmax_grad/mul" input: "gradients/bert/encoder/layer_7/attention/self/Softmax_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Softmax_grad/sub" op: "Sub" input: "gradients/bert/encoder/layer_7/attention/self/dropout/div_grad/Reshape" input: "gradients/bert/encoder/layer_7/attention/self/Softmax_grad/Sum" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Softmax_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_7/attention/self/Softmax_grad/sub" input: "bert/encoder/layer_7/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/value/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_7/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_7/attention/self/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\001\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_7/attention/self/add_grad/Shape" input: "gradients/bert/encoder/layer_7/attention/self/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_7/attention/self/Softmax_grad/mul_1" input: "gradients/bert/encoder/layer_7/attention/self/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/attention/self/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/self/add_grad/Sum" input: "gradients/bert/encoder/layer_7/attention/self/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_7/attention/self/Softmax_grad/mul_1" input: "gradients/bert/encoder/layer_7/attention/self/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/attention/self/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/self/add_grad/Sum_1" input: "gradients/bert/encoder/layer_7/attention/self/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/value/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_7/attention/self/Reshape_2_grad/Reshape" input: "bert/encoder/layer_7/attention/self/value/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_7/attention/self/value/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_6/output/LayerNorm/batchnorm/add_1" input: "gradients/bert/encoder/layer_7/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_7/attention/self/Mul_grad/Shape" input: "gradients/bert/encoder/layer_7/attention/self/Mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_7/attention/self/add_grad/Reshape" input: "bert/encoder/layer_7/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_7/attention/self/Mul_grad/Mul" input: "gradients/bert/encoder/layer_7/attention/self/Mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/self/Mul_grad/Sum" input: "gradients/bert/encoder/layer_7/attention/self/Mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_7/attention/self/MatMul" input: "gradients/bert/encoder/layer_7/attention/self/add_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_7/attention/self/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_7/attention/self/Mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/self/Mul_grad/Sum_1" input: "gradients/bert/encoder/layer_7/attention/self/Mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/MatMul_grad/MatMul" op: "BatchMatMul" input: "gradients/bert/encoder/layer_7/attention/self/Mul_grad/Reshape" input: "bert/encoder/layer_7/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/attention/self/MatMul_grad/MatMul_1" op: "BatchMatMul" input: "gradients/bert/encoder/layer_7/attention/self/Mul_grad/Reshape" input: "bert/encoder/layer_7/attention/self/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/attention/self/transpose_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_7/attention/self/transpose/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/transpose_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_7/attention/self/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_7/attention/self/transpose_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/transpose_1_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_7/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/transpose_1_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_7/attention/self/MatMul_grad/MatMul_1" input: "gradients/bert/encoder/layer_7/attention/self/transpose_1_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Reshape_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Reshape_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/self/transpose_grad/transpose" input: "gradients/bert/encoder/layer_7/attention/self/Reshape_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Reshape_1_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Reshape_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/self/transpose_1_grad/transpose" input: "gradients/bert/encoder/layer_7/attention/self/Reshape_1_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/query/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_7/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_7/attention/self/key/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_7/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_7/attention/self/query/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_7/attention/self/Reshape_grad/Reshape" input: "bert/encoder/layer_7/attention/self/query/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_7/attention/self/query/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_6/output/LayerNorm/batchnorm/add_1" input: "gradients/bert/encoder/layer_7/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/attention/self/key/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_7/attention/self/Reshape_1_grad/Reshape" input: "bert/encoder/layer_7/attention/self/key/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_7/attention/self/key/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_6/output/LayerNorm/batchnorm/add_1" input: "gradients/bert/encoder/layer_7/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_38" op: "AddN" input: "gradients/AddN_37" input: "gradients/bert/encoder/layer_7/attention/self/value/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_7/attention/self/query/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_7/attention/self/key/MatMul_grad/MatMul" attr { key: "N" value { i: 4 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_38" input: "bert/encoder/layer_6/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_38" input: "bert/encoder/layer_6/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Shape" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_38" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Sum_1" op: "Sum" input: "gradients/AddN_38" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Sum_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Neg" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Shape" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Reshape_1" input: "bert/encoder/layer_6/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_6/output/LayerNorm/moments/mean" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_39" op: "AddN" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Shape" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_39" input: "bert/encoder/layer_6/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_6/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/AddN_39" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_6/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/add_grad/Shape" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/add_grad/Sum_1" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/add" op: "Add" input: "bert/encoder/layer_6/output/LayerNorm/moments/variance/reduction_indices" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/add" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/range" op: "Range" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/range/start" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/Size" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/Shape_1" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/range" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/mod" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/DynamicStitch" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/add_grad/Reshape" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/Shape" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_6/output/add" input: "bert/encoder/layer_6/output/LayerNorm/moments/StopGradient" input: "^gradients/bert/encoder/layer_6/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/mul" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/add" op: "Add" input: "bert/encoder/layer_6/output/LayerNorm/moments/mean/reduction_indices" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/add" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/range" op: "Range" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/range/start" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/Size" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/Shape_1" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/range" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/mod" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/DynamicStitch" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_40" op: "AddN" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/Reshape" input: "gradients/bert/encoder/layer_6/output/LayerNorm/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/dropout/mul_grad/Mul" op: "Mul" input: "gradients/AddN_40" input: "bert/encoder/layer_6/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/dropout/mul_grad/Mul_1" op: "Mul" input: "gradients/AddN_40" input: "bert/encoder/layer_6/output/dropout/div" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/dropout/div_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/output/dropout/div_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_6/output/dropout/div_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_6/output/dropout/div_grad/Shape" input: "gradients/bert/encoder/layer_6/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/dropout/div_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_6/output/dropout/mul_grad/Mul" input: "bert/encoder/layer_6/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/dropout/div_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_6/output/dropout/div_grad/RealDiv" input: "gradients/bert/encoder/layer_6/output/dropout/div_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/output/dropout/div_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/output/dropout/div_grad/Sum" input: "gradients/bert/encoder/layer_6/output/dropout/div_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/dropout/div_grad/Neg" op: "Neg" input: "bert/encoder/layer_6/output/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/dropout/div_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_6/output/dropout/div_grad/Neg" input: "bert/encoder/layer_6/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/dropout/div_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_6/output/dropout/div_grad/RealDiv_1" input: "bert/encoder/layer_6/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/dropout/div_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_6/output/dropout/mul_grad/Mul" input: "gradients/bert/encoder/layer_6/output/dropout/div_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/dropout/div_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_6/output/dropout/div_grad/mul" input: "gradients/bert/encoder/layer_6/output/dropout/div_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/output/dropout/div_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_6/output/dropout/div_grad/Sum_1" input: "gradients/bert/encoder/layer_6/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_6/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_6/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_6/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_6/output/dropout/div_grad/Reshape" input: "bert/encoder/layer_6/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_6/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_6/intermediate/dense/mul_1" input: "gradients/bert/encoder/layer_6/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/mul_1_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_6/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_6/intermediate/dense/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/mul_1_grad/Mul_1" op: "Mul" input: "gradients/bert/encoder/layer_6/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_6/intermediate/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_6/intermediate/dense/mul_grad/Shape" input: "gradients/bert/encoder/layer_6/intermediate/dense/mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_6/intermediate/dense/mul_1_grad/Mul_1" input: "bert/encoder/layer_6/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_6/intermediate/dense/mul_grad/Mul" input: "gradients/bert/encoder/layer_6/intermediate/dense/mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/intermediate/dense/mul_grad/Sum" input: "gradients/bert/encoder/layer_6/intermediate/dense/mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_6/intermediate/dense/mul/x" input: "gradients/bert/encoder/layer_6/intermediate/dense/mul_1_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_6/intermediate/dense/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_6/intermediate/dense/mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_6/intermediate/dense/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_6/intermediate/dense/mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_6/intermediate/dense/add_grad/Shape" input: "gradients/bert/encoder/layer_6/intermediate/dense/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_6/intermediate/dense/mul_grad/Reshape_1" input: "gradients/bert/encoder/layer_6/intermediate/dense/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/intermediate/dense/add_grad/Sum" input: "gradients/bert/encoder/layer_6/intermediate/dense/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_6/intermediate/dense/mul_grad/Reshape_1" input: "gradients/bert/encoder/layer_6/intermediate/dense/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_6/intermediate/dense/add_grad/Sum_1" input: "gradients/bert/encoder/layer_6/intermediate/dense/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/Erf_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.128379225730896 } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/Erf_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_6/intermediate/dense/add_grad/Reshape_1" input: "gradients/bert/encoder/layer_6/intermediate/dense/Erf_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/Erf_grad/Square" op: "Square" input: "bert/encoder/layer_6/intermediate/dense/truediv" input: "^gradients/bert/encoder/layer_6/intermediate/dense/add_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/Erf_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_6/intermediate/dense/Erf_grad/Square" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/Erf_grad/Exp" op: "Exp" input: "gradients/bert/encoder/layer_6/intermediate/dense/Erf_grad/Neg" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/Erf_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_6/intermediate/dense/Erf_grad/mul" input: "gradients/bert/encoder/layer_6/intermediate/dense/Erf_grad/Exp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/truediv_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/truediv_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/truediv_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_6/intermediate/dense/truediv_grad/Shape" input: "gradients/bert/encoder/layer_6/intermediate/dense/truediv_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/truediv_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_6/intermediate/dense/Erf_grad/mul_1" input: "bert/encoder/layer_6/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/truediv_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_6/intermediate/dense/truediv_grad/RealDiv" input: "gradients/bert/encoder/layer_6/intermediate/dense/truediv_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/truediv_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/intermediate/dense/truediv_grad/Sum" input: "gradients/bert/encoder/layer_6/intermediate/dense/truediv_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/truediv_grad/Neg" op: "Neg" input: "bert/encoder/layer_6/intermediate/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/truediv_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_6/intermediate/dense/truediv_grad/Neg" input: "bert/encoder/layer_6/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/truediv_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_6/intermediate/dense/truediv_grad/RealDiv_1" input: "bert/encoder/layer_6/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/truediv_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_6/intermediate/dense/Erf_grad/mul_1" input: "gradients/bert/encoder/layer_6/intermediate/dense/truediv_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/truediv_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_6/intermediate/dense/truediv_grad/mul" input: "gradients/bert/encoder/layer_6/intermediate/dense/truediv_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/truediv_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_6/intermediate/dense/truediv_grad/Sum_1" input: "gradients/bert/encoder/layer_6/intermediate/dense/truediv_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/AddN_41" op: "AddN" input: "gradients/bert/encoder/layer_6/intermediate/dense/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_6/intermediate/dense/truediv_grad/Reshape" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/intermediate/dense/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/AddN_41" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/AddN_41" input: "bert/encoder/layer_6/intermediate/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_1" input: "gradients/AddN_41" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_42" op: "AddN" input: "gradients/AddN_40" input: "gradients/bert/encoder/layer_6/intermediate/dense/MatMul_grad/MatMul" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_42" input: "bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_42" input: "bert/encoder/layer_6/attention/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Shape" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_42" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Sum_1" op: "Sum" input: "gradients/AddN_42" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Sum_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Neg" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Reshape_1" input: "bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_6/attention/output/LayerNorm/moments/mean" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_43" op: "AddN" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Shape" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_43" input: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/AddN_43" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_grad/Shape" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_grad/Sum_1" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/add" op: "Add" input: "bert/encoder/layer_6/attention/output/LayerNorm/moments/variance/reduction_indices" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/add" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/range" op: "Range" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/range/start" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Size" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Shape_1" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/range" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/mod" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/DynamicStitch" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_grad/Reshape" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_6/attention/output/add" input: "bert/encoder/layer_6/attention/output/LayerNorm/moments/StopGradient" input: "^gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/mul" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/add" op: "Add" input: "bert/encoder/layer_6/attention/output/LayerNorm/moments/mean/reduction_indices" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/add" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/range" op: "Range" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/range/start" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Size" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Shape_1" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/range" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/mod" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/DynamicStitch" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_44" op: "AddN" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/dropout/mul_grad/Mul" op: "Mul" input: "gradients/AddN_44" input: "bert/encoder/layer_6/attention/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/dropout/mul_grad/Mul_1" op: "Mul" input: "gradients/AddN_44" input: "bert/encoder/layer_6/attention/output/dropout/div" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/dropout/div_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/dropout/div_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/dropout/div_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_6/attention/output/dropout/div_grad/Shape" input: "gradients/bert/encoder/layer_6/attention/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/dropout/div_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_6/attention/output/dropout/mul_grad/Mul" input: "bert/encoder/layer_6/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/dropout/div_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_6/attention/output/dropout/div_grad/RealDiv" input: "gradients/bert/encoder/layer_6/attention/output/dropout/div_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/attention/output/dropout/div_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/output/dropout/div_grad/Sum" input: "gradients/bert/encoder/layer_6/attention/output/dropout/div_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/dropout/div_grad/Neg" op: "Neg" input: "bert/encoder/layer_6/attention/output/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/dropout/div_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_6/attention/output/dropout/div_grad/Neg" input: "bert/encoder/layer_6/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/dropout/div_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_6/attention/output/dropout/div_grad/RealDiv_1" input: "bert/encoder/layer_6/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/dropout/div_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_6/attention/output/dropout/mul_grad/Mul" input: "gradients/bert/encoder/layer_6/attention/output/dropout/div_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/dropout/div_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_6/attention/output/dropout/div_grad/mul" input: "gradients/bert/encoder/layer_6/attention/output/dropout/div_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/attention/output/dropout/div_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/output/dropout/div_grad/Sum_1" input: "gradients/bert/encoder/layer_6/attention/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_6/attention/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_6/attention/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_6/attention/output/dropout/div_grad/Reshape" input: "bert/encoder/layer_6/attention/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_6/attention/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_6/attention/self/Reshape_3" input: "gradients/bert/encoder/layer_6/attention/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Reshape_3_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Reshape_3_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/output/dense/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_6/attention/self/Reshape_3_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/transpose_3_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_6/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/transpose_3_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_6/attention/self/Reshape_3_grad/Reshape" input: "gradients/bert/encoder/layer_6/attention/self/transpose_3_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/MatMul_1_grad/MatMul" op: "BatchMatMul" input: "gradients/bert/encoder/layer_6/attention/self/transpose_3_grad/transpose" input: "bert/encoder/layer_6/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "gradients/bert/encoder/layer_6/attention/self/MatMul_1_grad/MatMul_1" op: "BatchMatMul" input: "bert/encoder/layer_6/attention/self/dropout/mul" input: "gradients/bert/encoder/layer_6/attention/self/transpose_3_grad/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/attention/self/dropout/mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_6/attention/self/MatMul_1_grad/MatMul" input: "bert/encoder/layer_6/attention/self/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/dropout/mul_grad/Mul_1" op: "Mul" input: "gradients/bert/encoder/layer_6/attention/self/MatMul_1_grad/MatMul" input: "bert/encoder/layer_6/attention/self/dropout/div" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/transpose_2_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_6/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/transpose_2_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_6/attention/self/MatMul_1_grad/MatMul_1" input: "gradients/bert/encoder/layer_6/attention/self/transpose_2_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/dropout/div_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/dropout/div_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/dropout/div_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_6/attention/self/dropout/div_grad/Shape" input: "gradients/bert/encoder/layer_6/attention/self/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/dropout/div_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_6/attention/self/dropout/mul_grad/Mul" input: "bert/encoder/layer_6/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/dropout/div_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_6/attention/self/dropout/div_grad/RealDiv" input: "gradients/bert/encoder/layer_6/attention/self/dropout/div_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/attention/self/dropout/div_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/self/dropout/div_grad/Sum" input: "gradients/bert/encoder/layer_6/attention/self/dropout/div_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/dropout/div_grad/Neg" op: "Neg" input: "bert/encoder/layer_6/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/dropout/div_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_6/attention/self/dropout/div_grad/Neg" input: "bert/encoder/layer_6/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/dropout/div_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_6/attention/self/dropout/div_grad/RealDiv_1" input: "bert/encoder/layer_6/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/dropout/div_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_6/attention/self/dropout/mul_grad/Mul" input: "gradients/bert/encoder/layer_6/attention/self/dropout/div_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/dropout/div_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_6/attention/self/dropout/div_grad/mul" input: "gradients/bert/encoder/layer_6/attention/self/dropout/div_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/attention/self/dropout/div_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/self/dropout/div_grad/Sum_1" input: "gradients/bert/encoder/layer_6/attention/self/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Reshape_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Reshape_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/self/transpose_2_grad/transpose" input: "gradients/bert/encoder/layer_6/attention/self/Reshape_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Softmax_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_6/attention/self/dropout/div_grad/Reshape" input: "bert/encoder/layer_6/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Softmax_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: -1 } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Softmax_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_6/attention/self/Softmax_grad/mul" input: "gradients/bert/encoder/layer_6/attention/self/Softmax_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Softmax_grad/sub" op: "Sub" input: "gradients/bert/encoder/layer_6/attention/self/dropout/div_grad/Reshape" input: "gradients/bert/encoder/layer_6/attention/self/Softmax_grad/Sum" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Softmax_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_6/attention/self/Softmax_grad/sub" input: "bert/encoder/layer_6/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/value/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_6/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_6/attention/self/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\001\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_6/attention/self/add_grad/Shape" input: "gradients/bert/encoder/layer_6/attention/self/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_6/attention/self/Softmax_grad/mul_1" input: "gradients/bert/encoder/layer_6/attention/self/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/attention/self/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/self/add_grad/Sum" input: "gradients/bert/encoder/layer_6/attention/self/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_6/attention/self/Softmax_grad/mul_1" input: "gradients/bert/encoder/layer_6/attention/self/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/attention/self/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/self/add_grad/Sum_1" input: "gradients/bert/encoder/layer_6/attention/self/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/value/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_6/attention/self/Reshape_2_grad/Reshape" input: "bert/encoder/layer_6/attention/self/value/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_6/attention/self/value/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_5/output/LayerNorm/batchnorm/add_1" input: "gradients/bert/encoder/layer_6/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_6/attention/self/Mul_grad/Shape" input: "gradients/bert/encoder/layer_6/attention/self/Mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_6/attention/self/add_grad/Reshape" input: "bert/encoder/layer_6/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_6/attention/self/Mul_grad/Mul" input: "gradients/bert/encoder/layer_6/attention/self/Mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/self/Mul_grad/Sum" input: "gradients/bert/encoder/layer_6/attention/self/Mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_6/attention/self/MatMul" input: "gradients/bert/encoder/layer_6/attention/self/add_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_6/attention/self/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_6/attention/self/Mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/self/Mul_grad/Sum_1" input: "gradients/bert/encoder/layer_6/attention/self/Mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/MatMul_grad/MatMul" op: "BatchMatMul" input: "gradients/bert/encoder/layer_6/attention/self/Mul_grad/Reshape" input: "bert/encoder/layer_6/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/attention/self/MatMul_grad/MatMul_1" op: "BatchMatMul" input: "gradients/bert/encoder/layer_6/attention/self/Mul_grad/Reshape" input: "bert/encoder/layer_6/attention/self/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/attention/self/transpose_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_6/attention/self/transpose/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/transpose_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_6/attention/self/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_6/attention/self/transpose_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/transpose_1_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_6/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/transpose_1_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_6/attention/self/MatMul_grad/MatMul_1" input: "gradients/bert/encoder/layer_6/attention/self/transpose_1_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Reshape_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Reshape_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/self/transpose_grad/transpose" input: "gradients/bert/encoder/layer_6/attention/self/Reshape_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Reshape_1_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Reshape_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/self/transpose_1_grad/transpose" input: "gradients/bert/encoder/layer_6/attention/self/Reshape_1_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/query/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_6/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_6/attention/self/key/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_6/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_6/attention/self/query/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_6/attention/self/Reshape_grad/Reshape" input: "bert/encoder/layer_6/attention/self/query/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_6/attention/self/query/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_5/output/LayerNorm/batchnorm/add_1" input: "gradients/bert/encoder/layer_6/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/attention/self/key/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_6/attention/self/Reshape_1_grad/Reshape" input: "bert/encoder/layer_6/attention/self/key/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_6/attention/self/key/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_5/output/LayerNorm/batchnorm/add_1" input: "gradients/bert/encoder/layer_6/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_45" op: "AddN" input: "gradients/AddN_44" input: "gradients/bert/encoder/layer_6/attention/self/value/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_6/attention/self/query/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_6/attention/self/key/MatMul_grad/MatMul" attr { key: "N" value { i: 4 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_45" input: "bert/encoder/layer_5/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_45" input: "bert/encoder/layer_5/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Shape" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_45" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Sum_1" op: "Sum" input: "gradients/AddN_45" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Sum_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Neg" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Shape" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Reshape_1" input: "bert/encoder/layer_5/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_5/output/LayerNorm/moments/mean" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_46" op: "AddN" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Shape" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_46" input: "bert/encoder/layer_5/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_5/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/AddN_46" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_5/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/add_grad/Shape" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/add_grad/Sum_1" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/add" op: "Add" input: "bert/encoder/layer_5/output/LayerNorm/moments/variance/reduction_indices" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/add" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/range" op: "Range" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/range/start" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/Size" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/Shape_1" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/range" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/mod" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/DynamicStitch" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/add_grad/Reshape" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/Shape" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_5/output/add" input: "bert/encoder/layer_5/output/LayerNorm/moments/StopGradient" input: "^gradients/bert/encoder/layer_5/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/mul" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/add" op: "Add" input: "bert/encoder/layer_5/output/LayerNorm/moments/mean/reduction_indices" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/add" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/range" op: "Range" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/range/start" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/Size" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/Shape_1" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/range" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/mod" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/DynamicStitch" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_47" op: "AddN" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/Reshape" input: "gradients/bert/encoder/layer_5/output/LayerNorm/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/dropout/mul_grad/Mul" op: "Mul" input: "gradients/AddN_47" input: "bert/encoder/layer_5/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/dropout/mul_grad/Mul_1" op: "Mul" input: "gradients/AddN_47" input: "bert/encoder/layer_5/output/dropout/div" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/dropout/div_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/output/dropout/div_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_5/output/dropout/div_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_5/output/dropout/div_grad/Shape" input: "gradients/bert/encoder/layer_5/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/dropout/div_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_5/output/dropout/mul_grad/Mul" input: "bert/encoder/layer_5/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/dropout/div_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_5/output/dropout/div_grad/RealDiv" input: "gradients/bert/encoder/layer_5/output/dropout/div_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/output/dropout/div_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/output/dropout/div_grad/Sum" input: "gradients/bert/encoder/layer_5/output/dropout/div_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/dropout/div_grad/Neg" op: "Neg" input: "bert/encoder/layer_5/output/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/dropout/div_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_5/output/dropout/div_grad/Neg" input: "bert/encoder/layer_5/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/dropout/div_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_5/output/dropout/div_grad/RealDiv_1" input: "bert/encoder/layer_5/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/dropout/div_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_5/output/dropout/mul_grad/Mul" input: "gradients/bert/encoder/layer_5/output/dropout/div_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/dropout/div_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_5/output/dropout/div_grad/mul" input: "gradients/bert/encoder/layer_5/output/dropout/div_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/output/dropout/div_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_5/output/dropout/div_grad/Sum_1" input: "gradients/bert/encoder/layer_5/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_5/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_5/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_5/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_5/output/dropout/div_grad/Reshape" input: "bert/encoder/layer_5/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_5/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_5/intermediate/dense/mul_1" input: "gradients/bert/encoder/layer_5/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/mul_1_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_5/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_5/intermediate/dense/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/mul_1_grad/Mul_1" op: "Mul" input: "gradients/bert/encoder/layer_5/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_5/intermediate/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_5/intermediate/dense/mul_grad/Shape" input: "gradients/bert/encoder/layer_5/intermediate/dense/mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_5/intermediate/dense/mul_1_grad/Mul_1" input: "bert/encoder/layer_5/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_5/intermediate/dense/mul_grad/Mul" input: "gradients/bert/encoder/layer_5/intermediate/dense/mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/intermediate/dense/mul_grad/Sum" input: "gradients/bert/encoder/layer_5/intermediate/dense/mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_5/intermediate/dense/mul/x" input: "gradients/bert/encoder/layer_5/intermediate/dense/mul_1_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_5/intermediate/dense/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_5/intermediate/dense/mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_5/intermediate/dense/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_5/intermediate/dense/mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_5/intermediate/dense/add_grad/Shape" input: "gradients/bert/encoder/layer_5/intermediate/dense/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_5/intermediate/dense/mul_grad/Reshape_1" input: "gradients/bert/encoder/layer_5/intermediate/dense/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/intermediate/dense/add_grad/Sum" input: "gradients/bert/encoder/layer_5/intermediate/dense/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_5/intermediate/dense/mul_grad/Reshape_1" input: "gradients/bert/encoder/layer_5/intermediate/dense/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_5/intermediate/dense/add_grad/Sum_1" input: "gradients/bert/encoder/layer_5/intermediate/dense/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/Erf_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.128379225730896 } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/Erf_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_5/intermediate/dense/add_grad/Reshape_1" input: "gradients/bert/encoder/layer_5/intermediate/dense/Erf_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/Erf_grad/Square" op: "Square" input: "bert/encoder/layer_5/intermediate/dense/truediv" input: "^gradients/bert/encoder/layer_5/intermediate/dense/add_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/Erf_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_5/intermediate/dense/Erf_grad/Square" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/Erf_grad/Exp" op: "Exp" input: "gradients/bert/encoder/layer_5/intermediate/dense/Erf_grad/Neg" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/Erf_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_5/intermediate/dense/Erf_grad/mul" input: "gradients/bert/encoder/layer_5/intermediate/dense/Erf_grad/Exp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/truediv_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/truediv_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/truediv_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_5/intermediate/dense/truediv_grad/Shape" input: "gradients/bert/encoder/layer_5/intermediate/dense/truediv_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/truediv_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_5/intermediate/dense/Erf_grad/mul_1" input: "bert/encoder/layer_5/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/truediv_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_5/intermediate/dense/truediv_grad/RealDiv" input: "gradients/bert/encoder/layer_5/intermediate/dense/truediv_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/truediv_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/intermediate/dense/truediv_grad/Sum" input: "gradients/bert/encoder/layer_5/intermediate/dense/truediv_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/truediv_grad/Neg" op: "Neg" input: "bert/encoder/layer_5/intermediate/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/truediv_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_5/intermediate/dense/truediv_grad/Neg" input: "bert/encoder/layer_5/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/truediv_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_5/intermediate/dense/truediv_grad/RealDiv_1" input: "bert/encoder/layer_5/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/truediv_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_5/intermediate/dense/Erf_grad/mul_1" input: "gradients/bert/encoder/layer_5/intermediate/dense/truediv_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/truediv_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_5/intermediate/dense/truediv_grad/mul" input: "gradients/bert/encoder/layer_5/intermediate/dense/truediv_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/truediv_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_5/intermediate/dense/truediv_grad/Sum_1" input: "gradients/bert/encoder/layer_5/intermediate/dense/truediv_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/AddN_48" op: "AddN" input: "gradients/bert/encoder/layer_5/intermediate/dense/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_5/intermediate/dense/truediv_grad/Reshape" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/intermediate/dense/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/AddN_48" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/AddN_48" input: "bert/encoder/layer_5/intermediate/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_1" input: "gradients/AddN_48" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_49" op: "AddN" input: "gradients/AddN_47" input: "gradients/bert/encoder/layer_5/intermediate/dense/MatMul_grad/MatMul" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_49" input: "bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_49" input: "bert/encoder/layer_5/attention/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Shape" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_49" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Sum_1" op: "Sum" input: "gradients/AddN_49" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Sum_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Neg" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Reshape_1" input: "bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_5/attention/output/LayerNorm/moments/mean" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_50" op: "AddN" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Shape" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_50" input: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/AddN_50" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_grad/Shape" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_grad/Sum_1" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/add" op: "Add" input: "bert/encoder/layer_5/attention/output/LayerNorm/moments/variance/reduction_indices" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/add" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/range" op: "Range" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/range/start" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Size" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Shape_1" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/range" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/mod" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/DynamicStitch" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_grad/Reshape" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_5/attention/output/add" input: "bert/encoder/layer_5/attention/output/LayerNorm/moments/StopGradient" input: "^gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/mul" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/add" op: "Add" input: "bert/encoder/layer_5/attention/output/LayerNorm/moments/mean/reduction_indices" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/add" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/range" op: "Range" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/range/start" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Size" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Shape_1" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/range" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/mod" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/DynamicStitch" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_51" op: "AddN" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/dropout/mul_grad/Mul" op: "Mul" input: "gradients/AddN_51" input: "bert/encoder/layer_5/attention/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/dropout/mul_grad/Mul_1" op: "Mul" input: "gradients/AddN_51" input: "bert/encoder/layer_5/attention/output/dropout/div" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/dropout/div_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/dropout/div_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/dropout/div_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_5/attention/output/dropout/div_grad/Shape" input: "gradients/bert/encoder/layer_5/attention/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/dropout/div_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_5/attention/output/dropout/mul_grad/Mul" input: "bert/encoder/layer_5/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/dropout/div_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_5/attention/output/dropout/div_grad/RealDiv" input: "gradients/bert/encoder/layer_5/attention/output/dropout/div_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/attention/output/dropout/div_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/output/dropout/div_grad/Sum" input: "gradients/bert/encoder/layer_5/attention/output/dropout/div_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/dropout/div_grad/Neg" op: "Neg" input: "bert/encoder/layer_5/attention/output/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/dropout/div_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_5/attention/output/dropout/div_grad/Neg" input: "bert/encoder/layer_5/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/dropout/div_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_5/attention/output/dropout/div_grad/RealDiv_1" input: "bert/encoder/layer_5/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/dropout/div_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_5/attention/output/dropout/mul_grad/Mul" input: "gradients/bert/encoder/layer_5/attention/output/dropout/div_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/dropout/div_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_5/attention/output/dropout/div_grad/mul" input: "gradients/bert/encoder/layer_5/attention/output/dropout/div_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/attention/output/dropout/div_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/output/dropout/div_grad/Sum_1" input: "gradients/bert/encoder/layer_5/attention/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_5/attention/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_5/attention/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_5/attention/output/dropout/div_grad/Reshape" input: "bert/encoder/layer_5/attention/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_5/attention/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_5/attention/self/Reshape_3" input: "gradients/bert/encoder/layer_5/attention/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Reshape_3_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Reshape_3_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/output/dense/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_5/attention/self/Reshape_3_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/transpose_3_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_5/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/transpose_3_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_5/attention/self/Reshape_3_grad/Reshape" input: "gradients/bert/encoder/layer_5/attention/self/transpose_3_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/MatMul_1_grad/MatMul" op: "BatchMatMul" input: "gradients/bert/encoder/layer_5/attention/self/transpose_3_grad/transpose" input: "bert/encoder/layer_5/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "gradients/bert/encoder/layer_5/attention/self/MatMul_1_grad/MatMul_1" op: "BatchMatMul" input: "bert/encoder/layer_5/attention/self/dropout/mul" input: "gradients/bert/encoder/layer_5/attention/self/transpose_3_grad/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/attention/self/dropout/mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_5/attention/self/MatMul_1_grad/MatMul" input: "bert/encoder/layer_5/attention/self/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/dropout/mul_grad/Mul_1" op: "Mul" input: "gradients/bert/encoder/layer_5/attention/self/MatMul_1_grad/MatMul" input: "bert/encoder/layer_5/attention/self/dropout/div" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/transpose_2_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_5/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/transpose_2_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_5/attention/self/MatMul_1_grad/MatMul_1" input: "gradients/bert/encoder/layer_5/attention/self/transpose_2_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/dropout/div_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/dropout/div_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/dropout/div_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_5/attention/self/dropout/div_grad/Shape" input: "gradients/bert/encoder/layer_5/attention/self/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/dropout/div_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_5/attention/self/dropout/mul_grad/Mul" input: "bert/encoder/layer_5/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/dropout/div_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_5/attention/self/dropout/div_grad/RealDiv" input: "gradients/bert/encoder/layer_5/attention/self/dropout/div_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/attention/self/dropout/div_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/self/dropout/div_grad/Sum" input: "gradients/bert/encoder/layer_5/attention/self/dropout/div_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/dropout/div_grad/Neg" op: "Neg" input: "bert/encoder/layer_5/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/dropout/div_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_5/attention/self/dropout/div_grad/Neg" input: "bert/encoder/layer_5/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/dropout/div_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_5/attention/self/dropout/div_grad/RealDiv_1" input: "bert/encoder/layer_5/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/dropout/div_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_5/attention/self/dropout/mul_grad/Mul" input: "gradients/bert/encoder/layer_5/attention/self/dropout/div_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/dropout/div_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_5/attention/self/dropout/div_grad/mul" input: "gradients/bert/encoder/layer_5/attention/self/dropout/div_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/attention/self/dropout/div_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/self/dropout/div_grad/Sum_1" input: "gradients/bert/encoder/layer_5/attention/self/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Reshape_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Reshape_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/self/transpose_2_grad/transpose" input: "gradients/bert/encoder/layer_5/attention/self/Reshape_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Softmax_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_5/attention/self/dropout/div_grad/Reshape" input: "bert/encoder/layer_5/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Softmax_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: -1 } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Softmax_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_5/attention/self/Softmax_grad/mul" input: "gradients/bert/encoder/layer_5/attention/self/Softmax_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Softmax_grad/sub" op: "Sub" input: "gradients/bert/encoder/layer_5/attention/self/dropout/div_grad/Reshape" input: "gradients/bert/encoder/layer_5/attention/self/Softmax_grad/Sum" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Softmax_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_5/attention/self/Softmax_grad/sub" input: "bert/encoder/layer_5/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/value/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_5/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_5/attention/self/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\001\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_5/attention/self/add_grad/Shape" input: "gradients/bert/encoder/layer_5/attention/self/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_5/attention/self/Softmax_grad/mul_1" input: "gradients/bert/encoder/layer_5/attention/self/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/attention/self/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/self/add_grad/Sum" input: "gradients/bert/encoder/layer_5/attention/self/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_5/attention/self/Softmax_grad/mul_1" input: "gradients/bert/encoder/layer_5/attention/self/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/attention/self/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/self/add_grad/Sum_1" input: "gradients/bert/encoder/layer_5/attention/self/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/value/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_5/attention/self/Reshape_2_grad/Reshape" input: "bert/encoder/layer_5/attention/self/value/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_5/attention/self/value/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_4/output/LayerNorm/batchnorm/add_1" input: "gradients/bert/encoder/layer_5/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_5/attention/self/Mul_grad/Shape" input: "gradients/bert/encoder/layer_5/attention/self/Mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_5/attention/self/add_grad/Reshape" input: "bert/encoder/layer_5/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_5/attention/self/Mul_grad/Mul" input: "gradients/bert/encoder/layer_5/attention/self/Mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/self/Mul_grad/Sum" input: "gradients/bert/encoder/layer_5/attention/self/Mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_5/attention/self/MatMul" input: "gradients/bert/encoder/layer_5/attention/self/add_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_5/attention/self/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_5/attention/self/Mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/self/Mul_grad/Sum_1" input: "gradients/bert/encoder/layer_5/attention/self/Mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/MatMul_grad/MatMul" op: "BatchMatMul" input: "gradients/bert/encoder/layer_5/attention/self/Mul_grad/Reshape" input: "bert/encoder/layer_5/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/attention/self/MatMul_grad/MatMul_1" op: "BatchMatMul" input: "gradients/bert/encoder/layer_5/attention/self/Mul_grad/Reshape" input: "bert/encoder/layer_5/attention/self/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/attention/self/transpose_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_5/attention/self/transpose/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/transpose_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_5/attention/self/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_5/attention/self/transpose_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/transpose_1_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_5/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/transpose_1_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_5/attention/self/MatMul_grad/MatMul_1" input: "gradients/bert/encoder/layer_5/attention/self/transpose_1_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Reshape_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Reshape_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/self/transpose_grad/transpose" input: "gradients/bert/encoder/layer_5/attention/self/Reshape_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Reshape_1_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Reshape_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/self/transpose_1_grad/transpose" input: "gradients/bert/encoder/layer_5/attention/self/Reshape_1_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/query/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_5/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_5/attention/self/key/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_5/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_5/attention/self/query/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_5/attention/self/Reshape_grad/Reshape" input: "bert/encoder/layer_5/attention/self/query/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_5/attention/self/query/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_4/output/LayerNorm/batchnorm/add_1" input: "gradients/bert/encoder/layer_5/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/attention/self/key/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_5/attention/self/Reshape_1_grad/Reshape" input: "bert/encoder/layer_5/attention/self/key/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_5/attention/self/key/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_4/output/LayerNorm/batchnorm/add_1" input: "gradients/bert/encoder/layer_5/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_52" op: "AddN" input: "gradients/AddN_51" input: "gradients/bert/encoder/layer_5/attention/self/value/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_5/attention/self/query/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_5/attention/self/key/MatMul_grad/MatMul" attr { key: "N" value { i: 4 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_52" input: "bert/encoder/layer_4/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_52" input: "bert/encoder/layer_4/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Shape" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_52" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Sum_1" op: "Sum" input: "gradients/AddN_52" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Sum_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Neg" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Shape" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Reshape_1" input: "bert/encoder/layer_4/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_4/output/LayerNorm/moments/mean" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_53" op: "AddN" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Shape" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_53" input: "bert/encoder/layer_4/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_4/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/AddN_53" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_4/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/add_grad/Shape" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/add_grad/Sum_1" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/add" op: "Add" input: "bert/encoder/layer_4/output/LayerNorm/moments/variance/reduction_indices" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/add" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/range" op: "Range" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/range/start" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/Size" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/Shape_1" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/range" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/mod" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/DynamicStitch" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/add_grad/Reshape" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/Shape" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_4/output/add" input: "bert/encoder/layer_4/output/LayerNorm/moments/StopGradient" input: "^gradients/bert/encoder/layer_4/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/mul" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/add" op: "Add" input: "bert/encoder/layer_4/output/LayerNorm/moments/mean/reduction_indices" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/add" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/range" op: "Range" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/range/start" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/Size" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/Shape_1" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/range" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/mod" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/DynamicStitch" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_54" op: "AddN" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/Reshape" input: "gradients/bert/encoder/layer_4/output/LayerNorm/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/dropout/mul_grad/Mul" op: "Mul" input: "gradients/AddN_54" input: "bert/encoder/layer_4/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/dropout/mul_grad/Mul_1" op: "Mul" input: "gradients/AddN_54" input: "bert/encoder/layer_4/output/dropout/div" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/dropout/div_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/output/dropout/div_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_4/output/dropout/div_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_4/output/dropout/div_grad/Shape" input: "gradients/bert/encoder/layer_4/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/dropout/div_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_4/output/dropout/mul_grad/Mul" input: "bert/encoder/layer_4/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/dropout/div_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_4/output/dropout/div_grad/RealDiv" input: "gradients/bert/encoder/layer_4/output/dropout/div_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/output/dropout/div_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/output/dropout/div_grad/Sum" input: "gradients/bert/encoder/layer_4/output/dropout/div_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/dropout/div_grad/Neg" op: "Neg" input: "bert/encoder/layer_4/output/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/dropout/div_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_4/output/dropout/div_grad/Neg" input: "bert/encoder/layer_4/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/dropout/div_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_4/output/dropout/div_grad/RealDiv_1" input: "bert/encoder/layer_4/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/dropout/div_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_4/output/dropout/mul_grad/Mul" input: "gradients/bert/encoder/layer_4/output/dropout/div_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/dropout/div_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_4/output/dropout/div_grad/mul" input: "gradients/bert/encoder/layer_4/output/dropout/div_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/output/dropout/div_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_4/output/dropout/div_grad/Sum_1" input: "gradients/bert/encoder/layer_4/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_4/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_4/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_4/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_4/output/dropout/div_grad/Reshape" input: "bert/encoder/layer_4/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_4/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_4/intermediate/dense/mul_1" input: "gradients/bert/encoder/layer_4/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/mul_1_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_4/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_4/intermediate/dense/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/mul_1_grad/Mul_1" op: "Mul" input: "gradients/bert/encoder/layer_4/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_4/intermediate/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_4/intermediate/dense/mul_grad/Shape" input: "gradients/bert/encoder/layer_4/intermediate/dense/mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_4/intermediate/dense/mul_1_grad/Mul_1" input: "bert/encoder/layer_4/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_4/intermediate/dense/mul_grad/Mul" input: "gradients/bert/encoder/layer_4/intermediate/dense/mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/intermediate/dense/mul_grad/Sum" input: "gradients/bert/encoder/layer_4/intermediate/dense/mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_4/intermediate/dense/mul/x" input: "gradients/bert/encoder/layer_4/intermediate/dense/mul_1_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_4/intermediate/dense/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_4/intermediate/dense/mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_4/intermediate/dense/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_4/intermediate/dense/mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_4/intermediate/dense/add_grad/Shape" input: "gradients/bert/encoder/layer_4/intermediate/dense/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_4/intermediate/dense/mul_grad/Reshape_1" input: "gradients/bert/encoder/layer_4/intermediate/dense/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/intermediate/dense/add_grad/Sum" input: "gradients/bert/encoder/layer_4/intermediate/dense/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_4/intermediate/dense/mul_grad/Reshape_1" input: "gradients/bert/encoder/layer_4/intermediate/dense/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_4/intermediate/dense/add_grad/Sum_1" input: "gradients/bert/encoder/layer_4/intermediate/dense/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/Erf_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.128379225730896 } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/Erf_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_4/intermediate/dense/add_grad/Reshape_1" input: "gradients/bert/encoder/layer_4/intermediate/dense/Erf_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/Erf_grad/Square" op: "Square" input: "bert/encoder/layer_4/intermediate/dense/truediv" input: "^gradients/bert/encoder/layer_4/intermediate/dense/add_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/Erf_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_4/intermediate/dense/Erf_grad/Square" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/Erf_grad/Exp" op: "Exp" input: "gradients/bert/encoder/layer_4/intermediate/dense/Erf_grad/Neg" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/Erf_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_4/intermediate/dense/Erf_grad/mul" input: "gradients/bert/encoder/layer_4/intermediate/dense/Erf_grad/Exp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/truediv_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/truediv_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/truediv_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_4/intermediate/dense/truediv_grad/Shape" input: "gradients/bert/encoder/layer_4/intermediate/dense/truediv_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/truediv_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_4/intermediate/dense/Erf_grad/mul_1" input: "bert/encoder/layer_4/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/truediv_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_4/intermediate/dense/truediv_grad/RealDiv" input: "gradients/bert/encoder/layer_4/intermediate/dense/truediv_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/truediv_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/intermediate/dense/truediv_grad/Sum" input: "gradients/bert/encoder/layer_4/intermediate/dense/truediv_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/truediv_grad/Neg" op: "Neg" input: "bert/encoder/layer_4/intermediate/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/truediv_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_4/intermediate/dense/truediv_grad/Neg" input: "bert/encoder/layer_4/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/truediv_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_4/intermediate/dense/truediv_grad/RealDiv_1" input: "bert/encoder/layer_4/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/truediv_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_4/intermediate/dense/Erf_grad/mul_1" input: "gradients/bert/encoder/layer_4/intermediate/dense/truediv_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/truediv_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_4/intermediate/dense/truediv_grad/mul" input: "gradients/bert/encoder/layer_4/intermediate/dense/truediv_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/truediv_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_4/intermediate/dense/truediv_grad/Sum_1" input: "gradients/bert/encoder/layer_4/intermediate/dense/truediv_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/AddN_55" op: "AddN" input: "gradients/bert/encoder/layer_4/intermediate/dense/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_4/intermediate/dense/truediv_grad/Reshape" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/intermediate/dense/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/AddN_55" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/AddN_55" input: "bert/encoder/layer_4/intermediate/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_1" input: "gradients/AddN_55" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_56" op: "AddN" input: "gradients/AddN_54" input: "gradients/bert/encoder/layer_4/intermediate/dense/MatMul_grad/MatMul" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_56" input: "bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_56" input: "bert/encoder/layer_4/attention/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Shape" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_56" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Sum_1" op: "Sum" input: "gradients/AddN_56" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Sum_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Neg" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Reshape_1" input: "bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_4/attention/output/LayerNorm/moments/mean" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_57" op: "AddN" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Shape" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_57" input: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/AddN_57" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_grad/Shape" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_grad/Sum_1" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/add" op: "Add" input: "bert/encoder/layer_4/attention/output/LayerNorm/moments/variance/reduction_indices" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/add" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/range" op: "Range" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/range/start" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Size" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Shape_1" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/range" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/mod" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/DynamicStitch" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_grad/Reshape" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_4/attention/output/add" input: "bert/encoder/layer_4/attention/output/LayerNorm/moments/StopGradient" input: "^gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/mul" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/add" op: "Add" input: "bert/encoder/layer_4/attention/output/LayerNorm/moments/mean/reduction_indices" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/add" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/range" op: "Range" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/range/start" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Size" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Shape_1" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/range" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/mod" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/DynamicStitch" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_58" op: "AddN" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/dropout/mul_grad/Mul" op: "Mul" input: "gradients/AddN_58" input: "bert/encoder/layer_4/attention/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/dropout/mul_grad/Mul_1" op: "Mul" input: "gradients/AddN_58" input: "bert/encoder/layer_4/attention/output/dropout/div" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/dropout/div_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/dropout/div_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/dropout/div_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_4/attention/output/dropout/div_grad/Shape" input: "gradients/bert/encoder/layer_4/attention/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/dropout/div_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_4/attention/output/dropout/mul_grad/Mul" input: "bert/encoder/layer_4/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/dropout/div_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_4/attention/output/dropout/div_grad/RealDiv" input: "gradients/bert/encoder/layer_4/attention/output/dropout/div_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/attention/output/dropout/div_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/output/dropout/div_grad/Sum" input: "gradients/bert/encoder/layer_4/attention/output/dropout/div_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/dropout/div_grad/Neg" op: "Neg" input: "bert/encoder/layer_4/attention/output/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/dropout/div_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_4/attention/output/dropout/div_grad/Neg" input: "bert/encoder/layer_4/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/dropout/div_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_4/attention/output/dropout/div_grad/RealDiv_1" input: "bert/encoder/layer_4/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/dropout/div_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_4/attention/output/dropout/mul_grad/Mul" input: "gradients/bert/encoder/layer_4/attention/output/dropout/div_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/dropout/div_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_4/attention/output/dropout/div_grad/mul" input: "gradients/bert/encoder/layer_4/attention/output/dropout/div_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/attention/output/dropout/div_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/output/dropout/div_grad/Sum_1" input: "gradients/bert/encoder/layer_4/attention/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_4/attention/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_4/attention/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_4/attention/output/dropout/div_grad/Reshape" input: "bert/encoder/layer_4/attention/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_4/attention/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_4/attention/self/Reshape_3" input: "gradients/bert/encoder/layer_4/attention/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Reshape_3_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Reshape_3_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/output/dense/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_4/attention/self/Reshape_3_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/transpose_3_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_4/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/transpose_3_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_4/attention/self/Reshape_3_grad/Reshape" input: "gradients/bert/encoder/layer_4/attention/self/transpose_3_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/MatMul_1_grad/MatMul" op: "BatchMatMul" input: "gradients/bert/encoder/layer_4/attention/self/transpose_3_grad/transpose" input: "bert/encoder/layer_4/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "gradients/bert/encoder/layer_4/attention/self/MatMul_1_grad/MatMul_1" op: "BatchMatMul" input: "bert/encoder/layer_4/attention/self/dropout/mul" input: "gradients/bert/encoder/layer_4/attention/self/transpose_3_grad/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/attention/self/dropout/mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_4/attention/self/MatMul_1_grad/MatMul" input: "bert/encoder/layer_4/attention/self/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/dropout/mul_grad/Mul_1" op: "Mul" input: "gradients/bert/encoder/layer_4/attention/self/MatMul_1_grad/MatMul" input: "bert/encoder/layer_4/attention/self/dropout/div" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/transpose_2_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_4/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/transpose_2_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_4/attention/self/MatMul_1_grad/MatMul_1" input: "gradients/bert/encoder/layer_4/attention/self/transpose_2_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/dropout/div_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/dropout/div_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/dropout/div_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_4/attention/self/dropout/div_grad/Shape" input: "gradients/bert/encoder/layer_4/attention/self/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/dropout/div_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_4/attention/self/dropout/mul_grad/Mul" input: "bert/encoder/layer_4/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/dropout/div_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_4/attention/self/dropout/div_grad/RealDiv" input: "gradients/bert/encoder/layer_4/attention/self/dropout/div_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/attention/self/dropout/div_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/self/dropout/div_grad/Sum" input: "gradients/bert/encoder/layer_4/attention/self/dropout/div_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/dropout/div_grad/Neg" op: "Neg" input: "bert/encoder/layer_4/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/dropout/div_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_4/attention/self/dropout/div_grad/Neg" input: "bert/encoder/layer_4/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/dropout/div_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_4/attention/self/dropout/div_grad/RealDiv_1" input: "bert/encoder/layer_4/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/dropout/div_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_4/attention/self/dropout/mul_grad/Mul" input: "gradients/bert/encoder/layer_4/attention/self/dropout/div_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/dropout/div_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_4/attention/self/dropout/div_grad/mul" input: "gradients/bert/encoder/layer_4/attention/self/dropout/div_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/attention/self/dropout/div_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/self/dropout/div_grad/Sum_1" input: "gradients/bert/encoder/layer_4/attention/self/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Reshape_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Reshape_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/self/transpose_2_grad/transpose" input: "gradients/bert/encoder/layer_4/attention/self/Reshape_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Softmax_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_4/attention/self/dropout/div_grad/Reshape" input: "bert/encoder/layer_4/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Softmax_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: -1 } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Softmax_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_4/attention/self/Softmax_grad/mul" input: "gradients/bert/encoder/layer_4/attention/self/Softmax_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Softmax_grad/sub" op: "Sub" input: "gradients/bert/encoder/layer_4/attention/self/dropout/div_grad/Reshape" input: "gradients/bert/encoder/layer_4/attention/self/Softmax_grad/Sum" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Softmax_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_4/attention/self/Softmax_grad/sub" input: "bert/encoder/layer_4/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/value/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_4/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_4/attention/self/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\001\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_4/attention/self/add_grad/Shape" input: "gradients/bert/encoder/layer_4/attention/self/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_4/attention/self/Softmax_grad/mul_1" input: "gradients/bert/encoder/layer_4/attention/self/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/attention/self/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/self/add_grad/Sum" input: "gradients/bert/encoder/layer_4/attention/self/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_4/attention/self/Softmax_grad/mul_1" input: "gradients/bert/encoder/layer_4/attention/self/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/attention/self/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/self/add_grad/Sum_1" input: "gradients/bert/encoder/layer_4/attention/self/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/value/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_4/attention/self/Reshape_2_grad/Reshape" input: "bert/encoder/layer_4/attention/self/value/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_4/attention/self/value/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_3/output/LayerNorm/batchnorm/add_1" input: "gradients/bert/encoder/layer_4/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_4/attention/self/Mul_grad/Shape" input: "gradients/bert/encoder/layer_4/attention/self/Mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_4/attention/self/add_grad/Reshape" input: "bert/encoder/layer_4/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_4/attention/self/Mul_grad/Mul" input: "gradients/bert/encoder/layer_4/attention/self/Mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/self/Mul_grad/Sum" input: "gradients/bert/encoder/layer_4/attention/self/Mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_4/attention/self/MatMul" input: "gradients/bert/encoder/layer_4/attention/self/add_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_4/attention/self/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_4/attention/self/Mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/self/Mul_grad/Sum_1" input: "gradients/bert/encoder/layer_4/attention/self/Mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/MatMul_grad/MatMul" op: "BatchMatMul" input: "gradients/bert/encoder/layer_4/attention/self/Mul_grad/Reshape" input: "bert/encoder/layer_4/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/attention/self/MatMul_grad/MatMul_1" op: "BatchMatMul" input: "gradients/bert/encoder/layer_4/attention/self/Mul_grad/Reshape" input: "bert/encoder/layer_4/attention/self/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/attention/self/transpose_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_4/attention/self/transpose/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/transpose_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_4/attention/self/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_4/attention/self/transpose_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/transpose_1_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_4/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/transpose_1_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_4/attention/self/MatMul_grad/MatMul_1" input: "gradients/bert/encoder/layer_4/attention/self/transpose_1_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Reshape_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Reshape_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/self/transpose_grad/transpose" input: "gradients/bert/encoder/layer_4/attention/self/Reshape_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Reshape_1_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Reshape_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/self/transpose_1_grad/transpose" input: "gradients/bert/encoder/layer_4/attention/self/Reshape_1_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/query/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_4/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_4/attention/self/key/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_4/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_4/attention/self/query/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_4/attention/self/Reshape_grad/Reshape" input: "bert/encoder/layer_4/attention/self/query/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_4/attention/self/query/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_3/output/LayerNorm/batchnorm/add_1" input: "gradients/bert/encoder/layer_4/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/attention/self/key/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_4/attention/self/Reshape_1_grad/Reshape" input: "bert/encoder/layer_4/attention/self/key/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_4/attention/self/key/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_3/output/LayerNorm/batchnorm/add_1" input: "gradients/bert/encoder/layer_4/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_59" op: "AddN" input: "gradients/AddN_58" input: "gradients/bert/encoder/layer_4/attention/self/value/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_4/attention/self/query/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_4/attention/self/key/MatMul_grad/MatMul" attr { key: "N" value { i: 4 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_59" input: "bert/encoder/layer_3/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_59" input: "bert/encoder/layer_3/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Shape" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_59" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Sum_1" op: "Sum" input: "gradients/AddN_59" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Sum_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Neg" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Shape" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Reshape_1" input: "bert/encoder/layer_3/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_3/output/LayerNorm/moments/mean" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_60" op: "AddN" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Shape" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_60" input: "bert/encoder/layer_3/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_3/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/AddN_60" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_3/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/add_grad/Shape" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/add_grad/Sum_1" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/add" op: "Add" input: "bert/encoder/layer_3/output/LayerNorm/moments/variance/reduction_indices" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/add" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/range" op: "Range" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/range/start" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/Size" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/Shape_1" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/range" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/mod" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/DynamicStitch" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/add_grad/Reshape" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/Shape" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_3/output/add" input: "bert/encoder/layer_3/output/LayerNorm/moments/StopGradient" input: "^gradients/bert/encoder/layer_3/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/mul" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/add" op: "Add" input: "bert/encoder/layer_3/output/LayerNorm/moments/mean/reduction_indices" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/add" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/range" op: "Range" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/range/start" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/Size" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/Shape_1" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/range" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/mod" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/DynamicStitch" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_61" op: "AddN" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/Reshape" input: "gradients/bert/encoder/layer_3/output/LayerNorm/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/dropout/mul_grad/Mul" op: "Mul" input: "gradients/AddN_61" input: "bert/encoder/layer_3/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/dropout/mul_grad/Mul_1" op: "Mul" input: "gradients/AddN_61" input: "bert/encoder/layer_3/output/dropout/div" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/dropout/div_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/output/dropout/div_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_3/output/dropout/div_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_3/output/dropout/div_grad/Shape" input: "gradients/bert/encoder/layer_3/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/dropout/div_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_3/output/dropout/mul_grad/Mul" input: "bert/encoder/layer_3/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/dropout/div_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_3/output/dropout/div_grad/RealDiv" input: "gradients/bert/encoder/layer_3/output/dropout/div_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/output/dropout/div_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/output/dropout/div_grad/Sum" input: "gradients/bert/encoder/layer_3/output/dropout/div_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/dropout/div_grad/Neg" op: "Neg" input: "bert/encoder/layer_3/output/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/dropout/div_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_3/output/dropout/div_grad/Neg" input: "bert/encoder/layer_3/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/dropout/div_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_3/output/dropout/div_grad/RealDiv_1" input: "bert/encoder/layer_3/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/dropout/div_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_3/output/dropout/mul_grad/Mul" input: "gradients/bert/encoder/layer_3/output/dropout/div_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/dropout/div_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_3/output/dropout/div_grad/mul" input: "gradients/bert/encoder/layer_3/output/dropout/div_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/output/dropout/div_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_3/output/dropout/div_grad/Sum_1" input: "gradients/bert/encoder/layer_3/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_3/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_3/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_3/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_3/output/dropout/div_grad/Reshape" input: "bert/encoder/layer_3/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_3/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_3/intermediate/dense/mul_1" input: "gradients/bert/encoder/layer_3/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/mul_1_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_3/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_3/intermediate/dense/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/mul_1_grad/Mul_1" op: "Mul" input: "gradients/bert/encoder/layer_3/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_3/intermediate/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_3/intermediate/dense/mul_grad/Shape" input: "gradients/bert/encoder/layer_3/intermediate/dense/mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_3/intermediate/dense/mul_1_grad/Mul_1" input: "bert/encoder/layer_3/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_3/intermediate/dense/mul_grad/Mul" input: "gradients/bert/encoder/layer_3/intermediate/dense/mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/intermediate/dense/mul_grad/Sum" input: "gradients/bert/encoder/layer_3/intermediate/dense/mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_3/intermediate/dense/mul/x" input: "gradients/bert/encoder/layer_3/intermediate/dense/mul_1_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_3/intermediate/dense/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_3/intermediate/dense/mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_3/intermediate/dense/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_3/intermediate/dense/mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_3/intermediate/dense/add_grad/Shape" input: "gradients/bert/encoder/layer_3/intermediate/dense/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_3/intermediate/dense/mul_grad/Reshape_1" input: "gradients/bert/encoder/layer_3/intermediate/dense/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/intermediate/dense/add_grad/Sum" input: "gradients/bert/encoder/layer_3/intermediate/dense/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_3/intermediate/dense/mul_grad/Reshape_1" input: "gradients/bert/encoder/layer_3/intermediate/dense/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_3/intermediate/dense/add_grad/Sum_1" input: "gradients/bert/encoder/layer_3/intermediate/dense/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/Erf_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.128379225730896 } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/Erf_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_3/intermediate/dense/add_grad/Reshape_1" input: "gradients/bert/encoder/layer_3/intermediate/dense/Erf_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/Erf_grad/Square" op: "Square" input: "bert/encoder/layer_3/intermediate/dense/truediv" input: "^gradients/bert/encoder/layer_3/intermediate/dense/add_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/Erf_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_3/intermediate/dense/Erf_grad/Square" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/Erf_grad/Exp" op: "Exp" input: "gradients/bert/encoder/layer_3/intermediate/dense/Erf_grad/Neg" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/Erf_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_3/intermediate/dense/Erf_grad/mul" input: "gradients/bert/encoder/layer_3/intermediate/dense/Erf_grad/Exp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/truediv_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/truediv_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/truediv_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_3/intermediate/dense/truediv_grad/Shape" input: "gradients/bert/encoder/layer_3/intermediate/dense/truediv_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/truediv_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_3/intermediate/dense/Erf_grad/mul_1" input: "bert/encoder/layer_3/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/truediv_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_3/intermediate/dense/truediv_grad/RealDiv" input: "gradients/bert/encoder/layer_3/intermediate/dense/truediv_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/truediv_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/intermediate/dense/truediv_grad/Sum" input: "gradients/bert/encoder/layer_3/intermediate/dense/truediv_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/truediv_grad/Neg" op: "Neg" input: "bert/encoder/layer_3/intermediate/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/truediv_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_3/intermediate/dense/truediv_grad/Neg" input: "bert/encoder/layer_3/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/truediv_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_3/intermediate/dense/truediv_grad/RealDiv_1" input: "bert/encoder/layer_3/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/truediv_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_3/intermediate/dense/Erf_grad/mul_1" input: "gradients/bert/encoder/layer_3/intermediate/dense/truediv_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/truediv_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_3/intermediate/dense/truediv_grad/mul" input: "gradients/bert/encoder/layer_3/intermediate/dense/truediv_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/truediv_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_3/intermediate/dense/truediv_grad/Sum_1" input: "gradients/bert/encoder/layer_3/intermediate/dense/truediv_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/AddN_62" op: "AddN" input: "gradients/bert/encoder/layer_3/intermediate/dense/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_3/intermediate/dense/truediv_grad/Reshape" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/intermediate/dense/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/AddN_62" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/AddN_62" input: "bert/encoder/layer_3/intermediate/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_1" input: "gradients/AddN_62" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_63" op: "AddN" input: "gradients/AddN_61" input: "gradients/bert/encoder/layer_3/intermediate/dense/MatMul_grad/MatMul" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_63" input: "bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_63" input: "bert/encoder/layer_3/attention/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Shape" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_63" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Sum_1" op: "Sum" input: "gradients/AddN_63" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Sum_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Neg" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Reshape_1" input: "bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_3/attention/output/LayerNorm/moments/mean" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_64" op: "AddN" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Shape" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_64" input: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/AddN_64" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_grad/Shape" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_grad/Sum_1" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/add" op: "Add" input: "bert/encoder/layer_3/attention/output/LayerNorm/moments/variance/reduction_indices" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/add" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/range" op: "Range" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/range/start" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Size" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Shape_1" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/range" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/mod" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/DynamicStitch" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_grad/Reshape" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_3/attention/output/add" input: "bert/encoder/layer_3/attention/output/LayerNorm/moments/StopGradient" input: "^gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/mul" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/add" op: "Add" input: "bert/encoder/layer_3/attention/output/LayerNorm/moments/mean/reduction_indices" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/add" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/range" op: "Range" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/range/start" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Size" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Shape_1" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/range" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/mod" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/DynamicStitch" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_65" op: "AddN" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/dropout/mul_grad/Mul" op: "Mul" input: "gradients/AddN_65" input: "bert/encoder/layer_3/attention/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/dropout/mul_grad/Mul_1" op: "Mul" input: "gradients/AddN_65" input: "bert/encoder/layer_3/attention/output/dropout/div" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/dropout/div_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/dropout/div_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/dropout/div_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_3/attention/output/dropout/div_grad/Shape" input: "gradients/bert/encoder/layer_3/attention/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/dropout/div_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_3/attention/output/dropout/mul_grad/Mul" input: "bert/encoder/layer_3/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/dropout/div_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_3/attention/output/dropout/div_grad/RealDiv" input: "gradients/bert/encoder/layer_3/attention/output/dropout/div_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/attention/output/dropout/div_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/output/dropout/div_grad/Sum" input: "gradients/bert/encoder/layer_3/attention/output/dropout/div_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/dropout/div_grad/Neg" op: "Neg" input: "bert/encoder/layer_3/attention/output/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/dropout/div_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_3/attention/output/dropout/div_grad/Neg" input: "bert/encoder/layer_3/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/dropout/div_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_3/attention/output/dropout/div_grad/RealDiv_1" input: "bert/encoder/layer_3/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/dropout/div_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_3/attention/output/dropout/mul_grad/Mul" input: "gradients/bert/encoder/layer_3/attention/output/dropout/div_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/dropout/div_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_3/attention/output/dropout/div_grad/mul" input: "gradients/bert/encoder/layer_3/attention/output/dropout/div_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/attention/output/dropout/div_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/output/dropout/div_grad/Sum_1" input: "gradients/bert/encoder/layer_3/attention/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_3/attention/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_3/attention/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_3/attention/output/dropout/div_grad/Reshape" input: "bert/encoder/layer_3/attention/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_3/attention/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_3/attention/self/Reshape_3" input: "gradients/bert/encoder/layer_3/attention/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Reshape_3_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Reshape_3_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/output/dense/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_3/attention/self/Reshape_3_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/transpose_3_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_3/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/transpose_3_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_3/attention/self/Reshape_3_grad/Reshape" input: "gradients/bert/encoder/layer_3/attention/self/transpose_3_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/MatMul_1_grad/MatMul" op: "BatchMatMul" input: "gradients/bert/encoder/layer_3/attention/self/transpose_3_grad/transpose" input: "bert/encoder/layer_3/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "gradients/bert/encoder/layer_3/attention/self/MatMul_1_grad/MatMul_1" op: "BatchMatMul" input: "bert/encoder/layer_3/attention/self/dropout/mul" input: "gradients/bert/encoder/layer_3/attention/self/transpose_3_grad/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/attention/self/dropout/mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_3/attention/self/MatMul_1_grad/MatMul" input: "bert/encoder/layer_3/attention/self/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/dropout/mul_grad/Mul_1" op: "Mul" input: "gradients/bert/encoder/layer_3/attention/self/MatMul_1_grad/MatMul" input: "bert/encoder/layer_3/attention/self/dropout/div" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/transpose_2_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_3/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/transpose_2_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_3/attention/self/MatMul_1_grad/MatMul_1" input: "gradients/bert/encoder/layer_3/attention/self/transpose_2_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/dropout/div_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/dropout/div_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/dropout/div_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_3/attention/self/dropout/div_grad/Shape" input: "gradients/bert/encoder/layer_3/attention/self/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/dropout/div_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_3/attention/self/dropout/mul_grad/Mul" input: "bert/encoder/layer_3/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/dropout/div_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_3/attention/self/dropout/div_grad/RealDiv" input: "gradients/bert/encoder/layer_3/attention/self/dropout/div_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/attention/self/dropout/div_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/self/dropout/div_grad/Sum" input: "gradients/bert/encoder/layer_3/attention/self/dropout/div_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/dropout/div_grad/Neg" op: "Neg" input: "bert/encoder/layer_3/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/dropout/div_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_3/attention/self/dropout/div_grad/Neg" input: "bert/encoder/layer_3/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/dropout/div_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_3/attention/self/dropout/div_grad/RealDiv_1" input: "bert/encoder/layer_3/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/dropout/div_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_3/attention/self/dropout/mul_grad/Mul" input: "gradients/bert/encoder/layer_3/attention/self/dropout/div_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/dropout/div_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_3/attention/self/dropout/div_grad/mul" input: "gradients/bert/encoder/layer_3/attention/self/dropout/div_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/attention/self/dropout/div_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/self/dropout/div_grad/Sum_1" input: "gradients/bert/encoder/layer_3/attention/self/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Reshape_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Reshape_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/self/transpose_2_grad/transpose" input: "gradients/bert/encoder/layer_3/attention/self/Reshape_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Softmax_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_3/attention/self/dropout/div_grad/Reshape" input: "bert/encoder/layer_3/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Softmax_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: -1 } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Softmax_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_3/attention/self/Softmax_grad/mul" input: "gradients/bert/encoder/layer_3/attention/self/Softmax_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Softmax_grad/sub" op: "Sub" input: "gradients/bert/encoder/layer_3/attention/self/dropout/div_grad/Reshape" input: "gradients/bert/encoder/layer_3/attention/self/Softmax_grad/Sum" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Softmax_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_3/attention/self/Softmax_grad/sub" input: "bert/encoder/layer_3/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/value/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_3/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_3/attention/self/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\001\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_3/attention/self/add_grad/Shape" input: "gradients/bert/encoder/layer_3/attention/self/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_3/attention/self/Softmax_grad/mul_1" input: "gradients/bert/encoder/layer_3/attention/self/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/attention/self/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/self/add_grad/Sum" input: "gradients/bert/encoder/layer_3/attention/self/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_3/attention/self/Softmax_grad/mul_1" input: "gradients/bert/encoder/layer_3/attention/self/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/attention/self/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/self/add_grad/Sum_1" input: "gradients/bert/encoder/layer_3/attention/self/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/value/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_3/attention/self/Reshape_2_grad/Reshape" input: "bert/encoder/layer_3/attention/self/value/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_3/attention/self/value/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_2/output/LayerNorm/batchnorm/add_1" input: "gradients/bert/encoder/layer_3/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_3/attention/self/Mul_grad/Shape" input: "gradients/bert/encoder/layer_3/attention/self/Mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_3/attention/self/add_grad/Reshape" input: "bert/encoder/layer_3/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_3/attention/self/Mul_grad/Mul" input: "gradients/bert/encoder/layer_3/attention/self/Mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/self/Mul_grad/Sum" input: "gradients/bert/encoder/layer_3/attention/self/Mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_3/attention/self/MatMul" input: "gradients/bert/encoder/layer_3/attention/self/add_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_3/attention/self/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_3/attention/self/Mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/self/Mul_grad/Sum_1" input: "gradients/bert/encoder/layer_3/attention/self/Mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/MatMul_grad/MatMul" op: "BatchMatMul" input: "gradients/bert/encoder/layer_3/attention/self/Mul_grad/Reshape" input: "bert/encoder/layer_3/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/attention/self/MatMul_grad/MatMul_1" op: "BatchMatMul" input: "gradients/bert/encoder/layer_3/attention/self/Mul_grad/Reshape" input: "bert/encoder/layer_3/attention/self/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/attention/self/transpose_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_3/attention/self/transpose/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/transpose_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_3/attention/self/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_3/attention/self/transpose_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/transpose_1_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_3/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/transpose_1_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_3/attention/self/MatMul_grad/MatMul_1" input: "gradients/bert/encoder/layer_3/attention/self/transpose_1_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Reshape_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Reshape_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/self/transpose_grad/transpose" input: "gradients/bert/encoder/layer_3/attention/self/Reshape_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Reshape_1_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Reshape_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/self/transpose_1_grad/transpose" input: "gradients/bert/encoder/layer_3/attention/self/Reshape_1_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/query/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_3/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_3/attention/self/key/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_3/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_3/attention/self/query/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_3/attention/self/Reshape_grad/Reshape" input: "bert/encoder/layer_3/attention/self/query/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_3/attention/self/query/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_2/output/LayerNorm/batchnorm/add_1" input: "gradients/bert/encoder/layer_3/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/attention/self/key/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_3/attention/self/Reshape_1_grad/Reshape" input: "bert/encoder/layer_3/attention/self/key/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_3/attention/self/key/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_2/output/LayerNorm/batchnorm/add_1" input: "gradients/bert/encoder/layer_3/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_66" op: "AddN" input: "gradients/AddN_65" input: "gradients/bert/encoder/layer_3/attention/self/value/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_3/attention/self/query/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_3/attention/self/key/MatMul_grad/MatMul" attr { key: "N" value { i: 4 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_66" input: "bert/encoder/layer_2/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_66" input: "bert/encoder/layer_2/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Shape" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_66" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Sum_1" op: "Sum" input: "gradients/AddN_66" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Sum_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Neg" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Shape" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Reshape_1" input: "bert/encoder/layer_2/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_2/output/LayerNorm/moments/mean" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_67" op: "AddN" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Shape" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_67" input: "bert/encoder/layer_2/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_2/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/AddN_67" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_2/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/add_grad/Shape" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/add_grad/Sum_1" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/add" op: "Add" input: "bert/encoder/layer_2/output/LayerNorm/moments/variance/reduction_indices" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/add" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/range" op: "Range" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/range/start" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/Size" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/Shape_1" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/range" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/mod" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/DynamicStitch" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/add_grad/Reshape" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/Shape" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_2/output/add" input: "bert/encoder/layer_2/output/LayerNorm/moments/StopGradient" input: "^gradients/bert/encoder/layer_2/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/mul" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/add" op: "Add" input: "bert/encoder/layer_2/output/LayerNorm/moments/mean/reduction_indices" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/add" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/range" op: "Range" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/range/start" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/Size" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/Shape_1" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/range" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/mod" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/DynamicStitch" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_68" op: "AddN" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/Reshape" input: "gradients/bert/encoder/layer_2/output/LayerNorm/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/dropout/mul_grad/Mul" op: "Mul" input: "gradients/AddN_68" input: "bert/encoder/layer_2/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/dropout/mul_grad/Mul_1" op: "Mul" input: "gradients/AddN_68" input: "bert/encoder/layer_2/output/dropout/div" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/dropout/div_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/output/dropout/div_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_2/output/dropout/div_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_2/output/dropout/div_grad/Shape" input: "gradients/bert/encoder/layer_2/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/dropout/div_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_2/output/dropout/mul_grad/Mul" input: "bert/encoder/layer_2/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/dropout/div_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_2/output/dropout/div_grad/RealDiv" input: "gradients/bert/encoder/layer_2/output/dropout/div_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/output/dropout/div_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/output/dropout/div_grad/Sum" input: "gradients/bert/encoder/layer_2/output/dropout/div_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/dropout/div_grad/Neg" op: "Neg" input: "bert/encoder/layer_2/output/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/dropout/div_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_2/output/dropout/div_grad/Neg" input: "bert/encoder/layer_2/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/dropout/div_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_2/output/dropout/div_grad/RealDiv_1" input: "bert/encoder/layer_2/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/dropout/div_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_2/output/dropout/mul_grad/Mul" input: "gradients/bert/encoder/layer_2/output/dropout/div_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/dropout/div_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_2/output/dropout/div_grad/mul" input: "gradients/bert/encoder/layer_2/output/dropout/div_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/output/dropout/div_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_2/output/dropout/div_grad/Sum_1" input: "gradients/bert/encoder/layer_2/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_2/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_2/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_2/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_2/output/dropout/div_grad/Reshape" input: "bert/encoder/layer_2/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_2/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_2/intermediate/dense/mul_1" input: "gradients/bert/encoder/layer_2/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/mul_1_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_2/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_2/intermediate/dense/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/mul_1_grad/Mul_1" op: "Mul" input: "gradients/bert/encoder/layer_2/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_2/intermediate/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_2/intermediate/dense/mul_grad/Shape" input: "gradients/bert/encoder/layer_2/intermediate/dense/mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_2/intermediate/dense/mul_1_grad/Mul_1" input: "bert/encoder/layer_2/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_2/intermediate/dense/mul_grad/Mul" input: "gradients/bert/encoder/layer_2/intermediate/dense/mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/intermediate/dense/mul_grad/Sum" input: "gradients/bert/encoder/layer_2/intermediate/dense/mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_2/intermediate/dense/mul/x" input: "gradients/bert/encoder/layer_2/intermediate/dense/mul_1_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_2/intermediate/dense/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_2/intermediate/dense/mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_2/intermediate/dense/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_2/intermediate/dense/mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_2/intermediate/dense/add_grad/Shape" input: "gradients/bert/encoder/layer_2/intermediate/dense/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_2/intermediate/dense/mul_grad/Reshape_1" input: "gradients/bert/encoder/layer_2/intermediate/dense/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/intermediate/dense/add_grad/Sum" input: "gradients/bert/encoder/layer_2/intermediate/dense/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_2/intermediate/dense/mul_grad/Reshape_1" input: "gradients/bert/encoder/layer_2/intermediate/dense/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_2/intermediate/dense/add_grad/Sum_1" input: "gradients/bert/encoder/layer_2/intermediate/dense/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/Erf_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.128379225730896 } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/Erf_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_2/intermediate/dense/add_grad/Reshape_1" input: "gradients/bert/encoder/layer_2/intermediate/dense/Erf_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/Erf_grad/Square" op: "Square" input: "bert/encoder/layer_2/intermediate/dense/truediv" input: "^gradients/bert/encoder/layer_2/intermediate/dense/add_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/Erf_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_2/intermediate/dense/Erf_grad/Square" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/Erf_grad/Exp" op: "Exp" input: "gradients/bert/encoder/layer_2/intermediate/dense/Erf_grad/Neg" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/Erf_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_2/intermediate/dense/Erf_grad/mul" input: "gradients/bert/encoder/layer_2/intermediate/dense/Erf_grad/Exp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/truediv_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/truediv_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/truediv_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_2/intermediate/dense/truediv_grad/Shape" input: "gradients/bert/encoder/layer_2/intermediate/dense/truediv_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/truediv_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_2/intermediate/dense/Erf_grad/mul_1" input: "bert/encoder/layer_2/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/truediv_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_2/intermediate/dense/truediv_grad/RealDiv" input: "gradients/bert/encoder/layer_2/intermediate/dense/truediv_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/truediv_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/intermediate/dense/truediv_grad/Sum" input: "gradients/bert/encoder/layer_2/intermediate/dense/truediv_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/truediv_grad/Neg" op: "Neg" input: "bert/encoder/layer_2/intermediate/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/truediv_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_2/intermediate/dense/truediv_grad/Neg" input: "bert/encoder/layer_2/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/truediv_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_2/intermediate/dense/truediv_grad/RealDiv_1" input: "bert/encoder/layer_2/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/truediv_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_2/intermediate/dense/Erf_grad/mul_1" input: "gradients/bert/encoder/layer_2/intermediate/dense/truediv_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/truediv_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_2/intermediate/dense/truediv_grad/mul" input: "gradients/bert/encoder/layer_2/intermediate/dense/truediv_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/truediv_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_2/intermediate/dense/truediv_grad/Sum_1" input: "gradients/bert/encoder/layer_2/intermediate/dense/truediv_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/AddN_69" op: "AddN" input: "gradients/bert/encoder/layer_2/intermediate/dense/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_2/intermediate/dense/truediv_grad/Reshape" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/intermediate/dense/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/AddN_69" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/AddN_69" input: "bert/encoder/layer_2/intermediate/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_1" input: "gradients/AddN_69" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_70" op: "AddN" input: "gradients/AddN_68" input: "gradients/bert/encoder/layer_2/intermediate/dense/MatMul_grad/MatMul" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_70" input: "bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_70" input: "bert/encoder/layer_2/attention/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Shape" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_70" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Sum_1" op: "Sum" input: "gradients/AddN_70" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Sum_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Neg" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Reshape_1" input: "bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_2/attention/output/LayerNorm/moments/mean" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_71" op: "AddN" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Shape" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_71" input: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/AddN_71" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_grad/Shape" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_grad/Sum_1" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/add" op: "Add" input: "bert/encoder/layer_2/attention/output/LayerNorm/moments/variance/reduction_indices" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/add" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/range" op: "Range" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/range/start" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Size" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Shape_1" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/range" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/mod" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/DynamicStitch" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_grad/Reshape" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_2/attention/output/add" input: "bert/encoder/layer_2/attention/output/LayerNorm/moments/StopGradient" input: "^gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/mul" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/add" op: "Add" input: "bert/encoder/layer_2/attention/output/LayerNorm/moments/mean/reduction_indices" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/add" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/range" op: "Range" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/range/start" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Size" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Shape_1" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/range" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/mod" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/DynamicStitch" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_72" op: "AddN" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/dropout/mul_grad/Mul" op: "Mul" input: "gradients/AddN_72" input: "bert/encoder/layer_2/attention/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/dropout/mul_grad/Mul_1" op: "Mul" input: "gradients/AddN_72" input: "bert/encoder/layer_2/attention/output/dropout/div" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/dropout/div_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/dropout/div_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/dropout/div_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_2/attention/output/dropout/div_grad/Shape" input: "gradients/bert/encoder/layer_2/attention/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/dropout/div_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_2/attention/output/dropout/mul_grad/Mul" input: "bert/encoder/layer_2/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/dropout/div_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_2/attention/output/dropout/div_grad/RealDiv" input: "gradients/bert/encoder/layer_2/attention/output/dropout/div_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/attention/output/dropout/div_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/output/dropout/div_grad/Sum" input: "gradients/bert/encoder/layer_2/attention/output/dropout/div_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/dropout/div_grad/Neg" op: "Neg" input: "bert/encoder/layer_2/attention/output/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/dropout/div_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_2/attention/output/dropout/div_grad/Neg" input: "bert/encoder/layer_2/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/dropout/div_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_2/attention/output/dropout/div_grad/RealDiv_1" input: "bert/encoder/layer_2/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/dropout/div_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_2/attention/output/dropout/mul_grad/Mul" input: "gradients/bert/encoder/layer_2/attention/output/dropout/div_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/dropout/div_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_2/attention/output/dropout/div_grad/mul" input: "gradients/bert/encoder/layer_2/attention/output/dropout/div_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/attention/output/dropout/div_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/output/dropout/div_grad/Sum_1" input: "gradients/bert/encoder/layer_2/attention/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_2/attention/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_2/attention/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_2/attention/output/dropout/div_grad/Reshape" input: "bert/encoder/layer_2/attention/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_2/attention/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_2/attention/self/Reshape_3" input: "gradients/bert/encoder/layer_2/attention/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Reshape_3_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Reshape_3_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/output/dense/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_2/attention/self/Reshape_3_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/transpose_3_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_2/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/transpose_3_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_2/attention/self/Reshape_3_grad/Reshape" input: "gradients/bert/encoder/layer_2/attention/self/transpose_3_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/MatMul_1_grad/MatMul" op: "BatchMatMul" input: "gradients/bert/encoder/layer_2/attention/self/transpose_3_grad/transpose" input: "bert/encoder/layer_2/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "gradients/bert/encoder/layer_2/attention/self/MatMul_1_grad/MatMul_1" op: "BatchMatMul" input: "bert/encoder/layer_2/attention/self/dropout/mul" input: "gradients/bert/encoder/layer_2/attention/self/transpose_3_grad/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/attention/self/dropout/mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_2/attention/self/MatMul_1_grad/MatMul" input: "bert/encoder/layer_2/attention/self/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/dropout/mul_grad/Mul_1" op: "Mul" input: "gradients/bert/encoder/layer_2/attention/self/MatMul_1_grad/MatMul" input: "bert/encoder/layer_2/attention/self/dropout/div" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/transpose_2_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_2/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/transpose_2_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_2/attention/self/MatMul_1_grad/MatMul_1" input: "gradients/bert/encoder/layer_2/attention/self/transpose_2_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/dropout/div_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/dropout/div_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/dropout/div_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_2/attention/self/dropout/div_grad/Shape" input: "gradients/bert/encoder/layer_2/attention/self/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/dropout/div_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_2/attention/self/dropout/mul_grad/Mul" input: "bert/encoder/layer_2/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/dropout/div_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_2/attention/self/dropout/div_grad/RealDiv" input: "gradients/bert/encoder/layer_2/attention/self/dropout/div_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/attention/self/dropout/div_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/self/dropout/div_grad/Sum" input: "gradients/bert/encoder/layer_2/attention/self/dropout/div_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/dropout/div_grad/Neg" op: "Neg" input: "bert/encoder/layer_2/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/dropout/div_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_2/attention/self/dropout/div_grad/Neg" input: "bert/encoder/layer_2/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/dropout/div_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_2/attention/self/dropout/div_grad/RealDiv_1" input: "bert/encoder/layer_2/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/dropout/div_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_2/attention/self/dropout/mul_grad/Mul" input: "gradients/bert/encoder/layer_2/attention/self/dropout/div_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/dropout/div_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_2/attention/self/dropout/div_grad/mul" input: "gradients/bert/encoder/layer_2/attention/self/dropout/div_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/attention/self/dropout/div_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/self/dropout/div_grad/Sum_1" input: "gradients/bert/encoder/layer_2/attention/self/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Reshape_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Reshape_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/self/transpose_2_grad/transpose" input: "gradients/bert/encoder/layer_2/attention/self/Reshape_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Softmax_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_2/attention/self/dropout/div_grad/Reshape" input: "bert/encoder/layer_2/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Softmax_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: -1 } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Softmax_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_2/attention/self/Softmax_grad/mul" input: "gradients/bert/encoder/layer_2/attention/self/Softmax_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Softmax_grad/sub" op: "Sub" input: "gradients/bert/encoder/layer_2/attention/self/dropout/div_grad/Reshape" input: "gradients/bert/encoder/layer_2/attention/self/Softmax_grad/Sum" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Softmax_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_2/attention/self/Softmax_grad/sub" input: "bert/encoder/layer_2/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/value/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_2/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_2/attention/self/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\001\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_2/attention/self/add_grad/Shape" input: "gradients/bert/encoder/layer_2/attention/self/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_2/attention/self/Softmax_grad/mul_1" input: "gradients/bert/encoder/layer_2/attention/self/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/attention/self/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/self/add_grad/Sum" input: "gradients/bert/encoder/layer_2/attention/self/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_2/attention/self/Softmax_grad/mul_1" input: "gradients/bert/encoder/layer_2/attention/self/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/attention/self/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/self/add_grad/Sum_1" input: "gradients/bert/encoder/layer_2/attention/self/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/value/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_2/attention/self/Reshape_2_grad/Reshape" input: "bert/encoder/layer_2/attention/self/value/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_2/attention/self/value/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_1/output/LayerNorm/batchnorm/add_1" input: "gradients/bert/encoder/layer_2/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_2/attention/self/Mul_grad/Shape" input: "gradients/bert/encoder/layer_2/attention/self/Mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_2/attention/self/add_grad/Reshape" input: "bert/encoder/layer_2/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_2/attention/self/Mul_grad/Mul" input: "gradients/bert/encoder/layer_2/attention/self/Mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/self/Mul_grad/Sum" input: "gradients/bert/encoder/layer_2/attention/self/Mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_2/attention/self/MatMul" input: "gradients/bert/encoder/layer_2/attention/self/add_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_2/attention/self/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_2/attention/self/Mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/self/Mul_grad/Sum_1" input: "gradients/bert/encoder/layer_2/attention/self/Mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/MatMul_grad/MatMul" op: "BatchMatMul" input: "gradients/bert/encoder/layer_2/attention/self/Mul_grad/Reshape" input: "bert/encoder/layer_2/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/attention/self/MatMul_grad/MatMul_1" op: "BatchMatMul" input: "gradients/bert/encoder/layer_2/attention/self/Mul_grad/Reshape" input: "bert/encoder/layer_2/attention/self/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/attention/self/transpose_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_2/attention/self/transpose/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/transpose_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_2/attention/self/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_2/attention/self/transpose_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/transpose_1_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_2/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/transpose_1_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_2/attention/self/MatMul_grad/MatMul_1" input: "gradients/bert/encoder/layer_2/attention/self/transpose_1_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Reshape_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Reshape_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/self/transpose_grad/transpose" input: "gradients/bert/encoder/layer_2/attention/self/Reshape_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Reshape_1_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Reshape_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/self/transpose_1_grad/transpose" input: "gradients/bert/encoder/layer_2/attention/self/Reshape_1_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/query/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_2/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_2/attention/self/key/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_2/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_2/attention/self/query/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_2/attention/self/Reshape_grad/Reshape" input: "bert/encoder/layer_2/attention/self/query/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_2/attention/self/query/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_1/output/LayerNorm/batchnorm/add_1" input: "gradients/bert/encoder/layer_2/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/attention/self/key/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_2/attention/self/Reshape_1_grad/Reshape" input: "bert/encoder/layer_2/attention/self/key/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_2/attention/self/key/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_1/output/LayerNorm/batchnorm/add_1" input: "gradients/bert/encoder/layer_2/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_73" op: "AddN" input: "gradients/AddN_72" input: "gradients/bert/encoder/layer_2/attention/self/value/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_2/attention/self/query/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_2/attention/self/key/MatMul_grad/MatMul" attr { key: "N" value { i: 4 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_73" input: "bert/encoder/layer_1/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_73" input: "bert/encoder/layer_1/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Shape" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_73" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Sum_1" op: "Sum" input: "gradients/AddN_73" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Sum_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Neg" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Shape" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Reshape_1" input: "bert/encoder/layer_1/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_1/output/LayerNorm/moments/mean" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_74" op: "AddN" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Shape" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_74" input: "bert/encoder/layer_1/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_1/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/AddN_74" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_1/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/add_grad/Shape" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/add_grad/Sum_1" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/add" op: "Add" input: "bert/encoder/layer_1/output/LayerNorm/moments/variance/reduction_indices" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/add" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/range" op: "Range" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/range/start" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/Size" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/Shape_1" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/range" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/mod" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/DynamicStitch" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/add_grad/Reshape" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/Shape" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_1/output/add" input: "bert/encoder/layer_1/output/LayerNorm/moments/StopGradient" input: "^gradients/bert/encoder/layer_1/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/mul" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/add" op: "Add" input: "bert/encoder/layer_1/output/LayerNorm/moments/mean/reduction_indices" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/add" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/range" op: "Range" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/range/start" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/Size" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/Shape_1" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/range" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/mod" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/DynamicStitch" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_75" op: "AddN" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/Reshape" input: "gradients/bert/encoder/layer_1/output/LayerNorm/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/dropout/mul_grad/Mul" op: "Mul" input: "gradients/AddN_75" input: "bert/encoder/layer_1/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/dropout/mul_grad/Mul_1" op: "Mul" input: "gradients/AddN_75" input: "bert/encoder/layer_1/output/dropout/div" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/dropout/div_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/output/dropout/div_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_1/output/dropout/div_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_1/output/dropout/div_grad/Shape" input: "gradients/bert/encoder/layer_1/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/dropout/div_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_1/output/dropout/mul_grad/Mul" input: "bert/encoder/layer_1/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/dropout/div_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_1/output/dropout/div_grad/RealDiv" input: "gradients/bert/encoder/layer_1/output/dropout/div_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/output/dropout/div_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/output/dropout/div_grad/Sum" input: "gradients/bert/encoder/layer_1/output/dropout/div_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/dropout/div_grad/Neg" op: "Neg" input: "bert/encoder/layer_1/output/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/dropout/div_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_1/output/dropout/div_grad/Neg" input: "bert/encoder/layer_1/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/dropout/div_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_1/output/dropout/div_grad/RealDiv_1" input: "bert/encoder/layer_1/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/dropout/div_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_1/output/dropout/mul_grad/Mul" input: "gradients/bert/encoder/layer_1/output/dropout/div_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/dropout/div_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_1/output/dropout/div_grad/mul" input: "gradients/bert/encoder/layer_1/output/dropout/div_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/output/dropout/div_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_1/output/dropout/div_grad/Sum_1" input: "gradients/bert/encoder/layer_1/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_1/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_1/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_1/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_1/output/dropout/div_grad/Reshape" input: "bert/encoder/layer_1/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_1/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_1/intermediate/dense/mul_1" input: "gradients/bert/encoder/layer_1/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/mul_1_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_1/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_1/intermediate/dense/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/mul_1_grad/Mul_1" op: "Mul" input: "gradients/bert/encoder/layer_1/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_1/intermediate/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_1/intermediate/dense/mul_grad/Shape" input: "gradients/bert/encoder/layer_1/intermediate/dense/mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_1/intermediate/dense/mul_1_grad/Mul_1" input: "bert/encoder/layer_1/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_1/intermediate/dense/mul_grad/Mul" input: "gradients/bert/encoder/layer_1/intermediate/dense/mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/intermediate/dense/mul_grad/Sum" input: "gradients/bert/encoder/layer_1/intermediate/dense/mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_1/intermediate/dense/mul/x" input: "gradients/bert/encoder/layer_1/intermediate/dense/mul_1_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_1/intermediate/dense/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_1/intermediate/dense/mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_1/intermediate/dense/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_1/intermediate/dense/mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_1/intermediate/dense/add_grad/Shape" input: "gradients/bert/encoder/layer_1/intermediate/dense/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_1/intermediate/dense/mul_grad/Reshape_1" input: "gradients/bert/encoder/layer_1/intermediate/dense/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/intermediate/dense/add_grad/Sum" input: "gradients/bert/encoder/layer_1/intermediate/dense/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_1/intermediate/dense/mul_grad/Reshape_1" input: "gradients/bert/encoder/layer_1/intermediate/dense/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_1/intermediate/dense/add_grad/Sum_1" input: "gradients/bert/encoder/layer_1/intermediate/dense/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/Erf_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.128379225730896 } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/Erf_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_1/intermediate/dense/add_grad/Reshape_1" input: "gradients/bert/encoder/layer_1/intermediate/dense/Erf_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/Erf_grad/Square" op: "Square" input: "bert/encoder/layer_1/intermediate/dense/truediv" input: "^gradients/bert/encoder/layer_1/intermediate/dense/add_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/Erf_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_1/intermediate/dense/Erf_grad/Square" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/Erf_grad/Exp" op: "Exp" input: "gradients/bert/encoder/layer_1/intermediate/dense/Erf_grad/Neg" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/Erf_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_1/intermediate/dense/Erf_grad/mul" input: "gradients/bert/encoder/layer_1/intermediate/dense/Erf_grad/Exp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/truediv_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/truediv_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/truediv_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_1/intermediate/dense/truediv_grad/Shape" input: "gradients/bert/encoder/layer_1/intermediate/dense/truediv_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/truediv_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_1/intermediate/dense/Erf_grad/mul_1" input: "bert/encoder/layer_1/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/truediv_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_1/intermediate/dense/truediv_grad/RealDiv" input: "gradients/bert/encoder/layer_1/intermediate/dense/truediv_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/truediv_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/intermediate/dense/truediv_grad/Sum" input: "gradients/bert/encoder/layer_1/intermediate/dense/truediv_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/truediv_grad/Neg" op: "Neg" input: "bert/encoder/layer_1/intermediate/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/truediv_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_1/intermediate/dense/truediv_grad/Neg" input: "bert/encoder/layer_1/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/truediv_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_1/intermediate/dense/truediv_grad/RealDiv_1" input: "bert/encoder/layer_1/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/truediv_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_1/intermediate/dense/Erf_grad/mul_1" input: "gradients/bert/encoder/layer_1/intermediate/dense/truediv_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/truediv_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_1/intermediate/dense/truediv_grad/mul" input: "gradients/bert/encoder/layer_1/intermediate/dense/truediv_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/truediv_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_1/intermediate/dense/truediv_grad/Sum_1" input: "gradients/bert/encoder/layer_1/intermediate/dense/truediv_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/AddN_76" op: "AddN" input: "gradients/bert/encoder/layer_1/intermediate/dense/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_1/intermediate/dense/truediv_grad/Reshape" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/intermediate/dense/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/AddN_76" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/AddN_76" input: "bert/encoder/layer_1/intermediate/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_1" input: "gradients/AddN_76" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_77" op: "AddN" input: "gradients/AddN_75" input: "gradients/bert/encoder/layer_1/intermediate/dense/MatMul_grad/MatMul" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_77" input: "bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_77" input: "bert/encoder/layer_1/attention/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Shape" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_77" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Sum_1" op: "Sum" input: "gradients/AddN_77" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Sum_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Neg" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Reshape_1" input: "bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_1/attention/output/LayerNorm/moments/mean" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_78" op: "AddN" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Shape" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_78" input: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/AddN_78" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_grad/Shape" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_grad/Sum_1" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/add" op: "Add" input: "bert/encoder/layer_1/attention/output/LayerNorm/moments/variance/reduction_indices" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/add" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/range" op: "Range" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/range/start" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Size" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Shape_1" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/range" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/mod" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/DynamicStitch" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_grad/Reshape" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_1/attention/output/add" input: "bert/encoder/layer_1/attention/output/LayerNorm/moments/StopGradient" input: "^gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/mul" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/add" op: "Add" input: "bert/encoder/layer_1/attention/output/LayerNorm/moments/mean/reduction_indices" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/add" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/range" op: "Range" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/range/start" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Size" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Shape_1" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/range" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/mod" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/DynamicStitch" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_79" op: "AddN" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/dropout/mul_grad/Mul" op: "Mul" input: "gradients/AddN_79" input: "bert/encoder/layer_1/attention/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/dropout/mul_grad/Mul_1" op: "Mul" input: "gradients/AddN_79" input: "bert/encoder/layer_1/attention/output/dropout/div" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/dropout/div_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/dropout/div_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/dropout/div_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_1/attention/output/dropout/div_grad/Shape" input: "gradients/bert/encoder/layer_1/attention/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/dropout/div_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_1/attention/output/dropout/mul_grad/Mul" input: "bert/encoder/layer_1/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/dropout/div_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_1/attention/output/dropout/div_grad/RealDiv" input: "gradients/bert/encoder/layer_1/attention/output/dropout/div_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/attention/output/dropout/div_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/output/dropout/div_grad/Sum" input: "gradients/bert/encoder/layer_1/attention/output/dropout/div_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/dropout/div_grad/Neg" op: "Neg" input: "bert/encoder/layer_1/attention/output/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/dropout/div_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_1/attention/output/dropout/div_grad/Neg" input: "bert/encoder/layer_1/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/dropout/div_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_1/attention/output/dropout/div_grad/RealDiv_1" input: "bert/encoder/layer_1/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/dropout/div_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_1/attention/output/dropout/mul_grad/Mul" input: "gradients/bert/encoder/layer_1/attention/output/dropout/div_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/dropout/div_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_1/attention/output/dropout/div_grad/mul" input: "gradients/bert/encoder/layer_1/attention/output/dropout/div_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/attention/output/dropout/div_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/output/dropout/div_grad/Sum_1" input: "gradients/bert/encoder/layer_1/attention/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_1/attention/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_1/attention/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_1/attention/output/dropout/div_grad/Reshape" input: "bert/encoder/layer_1/attention/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_1/attention/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_1/attention/self/Reshape_3" input: "gradients/bert/encoder/layer_1/attention/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Reshape_3_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Reshape_3_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/output/dense/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_1/attention/self/Reshape_3_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/transpose_3_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_1/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/transpose_3_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_1/attention/self/Reshape_3_grad/Reshape" input: "gradients/bert/encoder/layer_1/attention/self/transpose_3_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/MatMul_1_grad/MatMul" op: "BatchMatMul" input: "gradients/bert/encoder/layer_1/attention/self/transpose_3_grad/transpose" input: "bert/encoder/layer_1/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "gradients/bert/encoder/layer_1/attention/self/MatMul_1_grad/MatMul_1" op: "BatchMatMul" input: "bert/encoder/layer_1/attention/self/dropout/mul" input: "gradients/bert/encoder/layer_1/attention/self/transpose_3_grad/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/attention/self/dropout/mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_1/attention/self/MatMul_1_grad/MatMul" input: "bert/encoder/layer_1/attention/self/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/dropout/mul_grad/Mul_1" op: "Mul" input: "gradients/bert/encoder/layer_1/attention/self/MatMul_1_grad/MatMul" input: "bert/encoder/layer_1/attention/self/dropout/div" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/transpose_2_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_1/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/transpose_2_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_1/attention/self/MatMul_1_grad/MatMul_1" input: "gradients/bert/encoder/layer_1/attention/self/transpose_2_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/dropout/div_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/dropout/div_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/dropout/div_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_1/attention/self/dropout/div_grad/Shape" input: "gradients/bert/encoder/layer_1/attention/self/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/dropout/div_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_1/attention/self/dropout/mul_grad/Mul" input: "bert/encoder/layer_1/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/dropout/div_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_1/attention/self/dropout/div_grad/RealDiv" input: "gradients/bert/encoder/layer_1/attention/self/dropout/div_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/attention/self/dropout/div_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/self/dropout/div_grad/Sum" input: "gradients/bert/encoder/layer_1/attention/self/dropout/div_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/dropout/div_grad/Neg" op: "Neg" input: "bert/encoder/layer_1/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/dropout/div_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_1/attention/self/dropout/div_grad/Neg" input: "bert/encoder/layer_1/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/dropout/div_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_1/attention/self/dropout/div_grad/RealDiv_1" input: "bert/encoder/layer_1/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/dropout/div_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_1/attention/self/dropout/mul_grad/Mul" input: "gradients/bert/encoder/layer_1/attention/self/dropout/div_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/dropout/div_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_1/attention/self/dropout/div_grad/mul" input: "gradients/bert/encoder/layer_1/attention/self/dropout/div_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/attention/self/dropout/div_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/self/dropout/div_grad/Sum_1" input: "gradients/bert/encoder/layer_1/attention/self/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Reshape_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Reshape_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/self/transpose_2_grad/transpose" input: "gradients/bert/encoder/layer_1/attention/self/Reshape_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Softmax_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_1/attention/self/dropout/div_grad/Reshape" input: "bert/encoder/layer_1/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Softmax_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: -1 } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Softmax_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_1/attention/self/Softmax_grad/mul" input: "gradients/bert/encoder/layer_1/attention/self/Softmax_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Softmax_grad/sub" op: "Sub" input: "gradients/bert/encoder/layer_1/attention/self/dropout/div_grad/Reshape" input: "gradients/bert/encoder/layer_1/attention/self/Softmax_grad/Sum" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Softmax_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_1/attention/self/Softmax_grad/sub" input: "bert/encoder/layer_1/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/value/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_1/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_1/attention/self/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\001\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_1/attention/self/add_grad/Shape" input: "gradients/bert/encoder/layer_1/attention/self/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_1/attention/self/Softmax_grad/mul_1" input: "gradients/bert/encoder/layer_1/attention/self/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/attention/self/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/self/add_grad/Sum" input: "gradients/bert/encoder/layer_1/attention/self/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_1/attention/self/Softmax_grad/mul_1" input: "gradients/bert/encoder/layer_1/attention/self/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/attention/self/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/self/add_grad/Sum_1" input: "gradients/bert/encoder/layer_1/attention/self/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/value/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_1/attention/self/Reshape_2_grad/Reshape" input: "bert/encoder/layer_1/attention/self/value/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_1/attention/self/value/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_0/output/LayerNorm/batchnorm/add_1" input: "gradients/bert/encoder/layer_1/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_1/attention/self/Mul_grad/Shape" input: "gradients/bert/encoder/layer_1/attention/self/Mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_1/attention/self/add_grad/Reshape" input: "bert/encoder/layer_1/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_1/attention/self/Mul_grad/Mul" input: "gradients/bert/encoder/layer_1/attention/self/Mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/self/Mul_grad/Sum" input: "gradients/bert/encoder/layer_1/attention/self/Mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_1/attention/self/MatMul" input: "gradients/bert/encoder/layer_1/attention/self/add_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_1/attention/self/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_1/attention/self/Mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/self/Mul_grad/Sum_1" input: "gradients/bert/encoder/layer_1/attention/self/Mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/MatMul_grad/MatMul" op: "BatchMatMul" input: "gradients/bert/encoder/layer_1/attention/self/Mul_grad/Reshape" input: "bert/encoder/layer_1/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/attention/self/MatMul_grad/MatMul_1" op: "BatchMatMul" input: "gradients/bert/encoder/layer_1/attention/self/Mul_grad/Reshape" input: "bert/encoder/layer_1/attention/self/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/attention/self/transpose_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_1/attention/self/transpose/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/transpose_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_1/attention/self/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_1/attention/self/transpose_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/transpose_1_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_1/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/transpose_1_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_1/attention/self/MatMul_grad/MatMul_1" input: "gradients/bert/encoder/layer_1/attention/self/transpose_1_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Reshape_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Reshape_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/self/transpose_grad/transpose" input: "gradients/bert/encoder/layer_1/attention/self/Reshape_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Reshape_1_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Reshape_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/self/transpose_1_grad/transpose" input: "gradients/bert/encoder/layer_1/attention/self/Reshape_1_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/query/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_1/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_1/attention/self/key/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_1/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_1/attention/self/query/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_1/attention/self/Reshape_grad/Reshape" input: "bert/encoder/layer_1/attention/self/query/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_1/attention/self/query/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_0/output/LayerNorm/batchnorm/add_1" input: "gradients/bert/encoder/layer_1/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/attention/self/key/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_1/attention/self/Reshape_1_grad/Reshape" input: "bert/encoder/layer_1/attention/self/key/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_1/attention/self/key/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_0/output/LayerNorm/batchnorm/add_1" input: "gradients/bert/encoder/layer_1/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_80" op: "AddN" input: "gradients/AddN_79" input: "gradients/bert/encoder/layer_1/attention/self/value/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_1/attention/self/query/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_1/attention/self/key/MatMul_grad/MatMul" attr { key: "N" value { i: 4 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_80" input: "bert/encoder/layer_0/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_80" input: "bert/encoder/layer_0/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Shape" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_80" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Sum_1" op: "Sum" input: "gradients/AddN_80" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Sum_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Neg" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Shape" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Reshape_1" input: "bert/encoder/layer_0/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_0/output/LayerNorm/moments/mean" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_81" op: "AddN" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Shape" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_81" input: "bert/encoder/layer_0/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_0/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/AddN_81" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_0/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/add_grad/Shape" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/add_grad/Sum_1" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/add" op: "Add" input: "bert/encoder/layer_0/output/LayerNorm/moments/variance/reduction_indices" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/add" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/range" op: "Range" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/range/start" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/Size" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/Shape_1" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/range" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/mod" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/DynamicStitch" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/add_grad/Reshape" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/Shape" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_0/output/add" input: "bert/encoder/layer_0/output/LayerNorm/moments/StopGradient" input: "^gradients/bert/encoder/layer_0/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/mul" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/add" op: "Add" input: "bert/encoder/layer_0/output/LayerNorm/moments/mean/reduction_indices" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/add" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/range" op: "Range" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/range/start" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/Size" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/Shape_1" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/range" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/mod" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/DynamicStitch" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_82" op: "AddN" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/Reshape" input: "gradients/bert/encoder/layer_0/output/LayerNorm/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/dropout/mul_grad/Mul" op: "Mul" input: "gradients/AddN_82" input: "bert/encoder/layer_0/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/dropout/mul_grad/Mul_1" op: "Mul" input: "gradients/AddN_82" input: "bert/encoder/layer_0/output/dropout/div" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/dropout/div_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/output/dropout/div_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_0/output/dropout/div_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_0/output/dropout/div_grad/Shape" input: "gradients/bert/encoder/layer_0/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/dropout/div_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_0/output/dropout/mul_grad/Mul" input: "bert/encoder/layer_0/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/dropout/div_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_0/output/dropout/div_grad/RealDiv" input: "gradients/bert/encoder/layer_0/output/dropout/div_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/output/dropout/div_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/output/dropout/div_grad/Sum" input: "gradients/bert/encoder/layer_0/output/dropout/div_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/dropout/div_grad/Neg" op: "Neg" input: "bert/encoder/layer_0/output/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/dropout/div_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_0/output/dropout/div_grad/Neg" input: "bert/encoder/layer_0/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/dropout/div_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_0/output/dropout/div_grad/RealDiv_1" input: "bert/encoder/layer_0/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/dropout/div_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_0/output/dropout/mul_grad/Mul" input: "gradients/bert/encoder/layer_0/output/dropout/div_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/dropout/div_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_0/output/dropout/div_grad/mul" input: "gradients/bert/encoder/layer_0/output/dropout/div_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/output/dropout/div_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_0/output/dropout/div_grad/Sum_1" input: "gradients/bert/encoder/layer_0/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_0/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_0/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_0/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_0/output/dropout/div_grad/Reshape" input: "bert/encoder/layer_0/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_0/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_0/intermediate/dense/mul_1" input: "gradients/bert/encoder/layer_0/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/mul_1_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_0/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_0/intermediate/dense/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/mul_1_grad/Mul_1" op: "Mul" input: "gradients/bert/encoder/layer_0/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_0/intermediate/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_0/intermediate/dense/mul_grad/Shape" input: "gradients/bert/encoder/layer_0/intermediate/dense/mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_0/intermediate/dense/mul_1_grad/Mul_1" input: "bert/encoder/layer_0/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_0/intermediate/dense/mul_grad/Mul" input: "gradients/bert/encoder/layer_0/intermediate/dense/mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/intermediate/dense/mul_grad/Sum" input: "gradients/bert/encoder/layer_0/intermediate/dense/mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_0/intermediate/dense/mul/x" input: "gradients/bert/encoder/layer_0/intermediate/dense/mul_1_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_0/intermediate/dense/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_0/intermediate/dense/mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_0/intermediate/dense/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_0/intermediate/dense/mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_0/intermediate/dense/add_grad/Shape" input: "gradients/bert/encoder/layer_0/intermediate/dense/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_0/intermediate/dense/mul_grad/Reshape_1" input: "gradients/bert/encoder/layer_0/intermediate/dense/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/intermediate/dense/add_grad/Sum" input: "gradients/bert/encoder/layer_0/intermediate/dense/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_0/intermediate/dense/mul_grad/Reshape_1" input: "gradients/bert/encoder/layer_0/intermediate/dense/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_0/intermediate/dense/add_grad/Sum_1" input: "gradients/bert/encoder/layer_0/intermediate/dense/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/Erf_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.128379225730896 } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/Erf_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_0/intermediate/dense/add_grad/Reshape_1" input: "gradients/bert/encoder/layer_0/intermediate/dense/Erf_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/Erf_grad/Square" op: "Square" input: "bert/encoder/layer_0/intermediate/dense/truediv" input: "^gradients/bert/encoder/layer_0/intermediate/dense/add_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/Erf_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_0/intermediate/dense/Erf_grad/Square" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/Erf_grad/Exp" op: "Exp" input: "gradients/bert/encoder/layer_0/intermediate/dense/Erf_grad/Neg" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/Erf_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_0/intermediate/dense/Erf_grad/mul" input: "gradients/bert/encoder/layer_0/intermediate/dense/Erf_grad/Exp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/truediv_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/truediv_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/truediv_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_0/intermediate/dense/truediv_grad/Shape" input: "gradients/bert/encoder/layer_0/intermediate/dense/truediv_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/truediv_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_0/intermediate/dense/Erf_grad/mul_1" input: "bert/encoder/layer_0/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/truediv_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_0/intermediate/dense/truediv_grad/RealDiv" input: "gradients/bert/encoder/layer_0/intermediate/dense/truediv_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/truediv_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/intermediate/dense/truediv_grad/Sum" input: "gradients/bert/encoder/layer_0/intermediate/dense/truediv_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/truediv_grad/Neg" op: "Neg" input: "bert/encoder/layer_0/intermediate/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/truediv_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_0/intermediate/dense/truediv_grad/Neg" input: "bert/encoder/layer_0/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/truediv_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_0/intermediate/dense/truediv_grad/RealDiv_1" input: "bert/encoder/layer_0/intermediate/dense/Sqrt" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/truediv_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_0/intermediate/dense/Erf_grad/mul_1" input: "gradients/bert/encoder/layer_0/intermediate/dense/truediv_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/truediv_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_0/intermediate/dense/truediv_grad/mul" input: "gradients/bert/encoder/layer_0/intermediate/dense/truediv_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/truediv_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_0/intermediate/dense/truediv_grad/Sum_1" input: "gradients/bert/encoder/layer_0/intermediate/dense/truediv_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/AddN_83" op: "AddN" input: "gradients/bert/encoder/layer_0/intermediate/dense/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_0/intermediate/dense/truediv_grad/Reshape" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/intermediate/dense/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/AddN_83" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/AddN_83" input: "bert/encoder/layer_0/intermediate/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_1" input: "gradients/AddN_83" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_84" op: "AddN" input: "gradients/AddN_82" input: "gradients/bert/encoder/layer_0/intermediate/dense/MatMul_grad/MatMul" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_84" input: "bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_84" input: "bert/encoder/layer_0/attention/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Shape" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_84" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Sum_1" op: "Sum" input: "gradients/AddN_84" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Sum_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Neg" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Reshape_1" input: "bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_0/attention/output/LayerNorm/moments/mean" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum_1" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_85" op: "AddN" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Shape" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_85" input: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/AddN_85" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_grad/Shape" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_grad/Sum_1" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/add" op: "Add" input: "bert/encoder/layer_0/attention/output/LayerNorm/moments/variance/reduction_indices" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/add" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/range" op: "Range" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/range/start" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Size" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Shape_1" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/range" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/mod" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/DynamicStitch" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_grad/Reshape" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_0/attention/output/add" input: "bert/encoder/layer_0/attention/output/LayerNorm/moments/StopGradient" input: "^gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/mul" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum_1" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/add" op: "Add" input: "bert/encoder/layer_0/attention/output/LayerNorm/moments/mean/reduction_indices" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/mod" op: "FloorMod" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/add" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/range" op: "Range" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/range/start" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Size" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Fill" op: "Fill" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Shape_1" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/range" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/mod" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/DynamicStitch" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/AddN_86" op: "AddN" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/dropout/mul_grad/Mul" op: "Mul" input: "gradients/AddN_86" input: "bert/encoder/layer_0/attention/output/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/dropout/mul_grad/Mul_1" op: "Mul" input: "gradients/AddN_86" input: "bert/encoder/layer_0/attention/output/dropout/div" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/dropout/div_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/dropout/div_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/dropout/div_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_0/attention/output/dropout/div_grad/Shape" input: "gradients/bert/encoder/layer_0/attention/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/dropout/div_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_0/attention/output/dropout/mul_grad/Mul" input: "bert/encoder/layer_0/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/dropout/div_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_0/attention/output/dropout/div_grad/RealDiv" input: "gradients/bert/encoder/layer_0/attention/output/dropout/div_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/attention/output/dropout/div_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/output/dropout/div_grad/Sum" input: "gradients/bert/encoder/layer_0/attention/output/dropout/div_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/dropout/div_grad/Neg" op: "Neg" input: "bert/encoder/layer_0/attention/output/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/dropout/div_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_0/attention/output/dropout/div_grad/Neg" input: "bert/encoder/layer_0/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/dropout/div_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_0/attention/output/dropout/div_grad/RealDiv_1" input: "bert/encoder/layer_0/attention/output/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/dropout/div_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_0/attention/output/dropout/mul_grad/Mul" input: "gradients/bert/encoder/layer_0/attention/output/dropout/div_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/dropout/div_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_0/attention/output/dropout/div_grad/mul" input: "gradients/bert/encoder/layer_0/attention/output/dropout/div_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/attention/output/dropout/div_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/output/dropout/div_grad/Sum_1" input: "gradients/bert/encoder/layer_0/attention/output/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_0/attention/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_0/attention/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_0/attention/output/dropout/div_grad/Reshape" input: "bert/encoder/layer_0/attention/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_0/attention/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_0/attention/self/Reshape_3" input: "gradients/bert/encoder/layer_0/attention/output/dropout/div_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Reshape_3_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Reshape_3_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/output/dense/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_0/attention/self/Reshape_3_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/transpose_3_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_0/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/transpose_3_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_0/attention/self/Reshape_3_grad/Reshape" input: "gradients/bert/encoder/layer_0/attention/self/transpose_3_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/MatMul_1_grad/MatMul" op: "BatchMatMul" input: "gradients/bert/encoder/layer_0/attention/self/transpose_3_grad/transpose" input: "bert/encoder/layer_0/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "gradients/bert/encoder/layer_0/attention/self/MatMul_1_grad/MatMul_1" op: "BatchMatMul" input: "bert/encoder/layer_0/attention/self/dropout/mul" input: "gradients/bert/encoder/layer_0/attention/self/transpose_3_grad/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/attention/self/dropout/mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_0/attention/self/MatMul_1_grad/MatMul" input: "bert/encoder/layer_0/attention/self/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/dropout/mul_grad/Mul_1" op: "Mul" input: "gradients/bert/encoder/layer_0/attention/self/MatMul_1_grad/MatMul" input: "bert/encoder/layer_0/attention/self/dropout/div" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/transpose_2_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_0/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/transpose_2_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_0/attention/self/MatMul_1_grad/MatMul_1" input: "gradients/bert/encoder/layer_0/attention/self/transpose_2_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/dropout/div_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/dropout/div_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/dropout/div_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_0/attention/self/dropout/div_grad/Shape" input: "gradients/bert/encoder/layer_0/attention/self/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/dropout/div_grad/RealDiv" op: "RealDiv" input: "gradients/bert/encoder/layer_0/attention/self/dropout/mul_grad/Mul" input: "bert/encoder/layer_0/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/dropout/div_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_0/attention/self/dropout/div_grad/RealDiv" input: "gradients/bert/encoder/layer_0/attention/self/dropout/div_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/attention/self/dropout/div_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/self/dropout/div_grad/Sum" input: "gradients/bert/encoder/layer_0/attention/self/dropout/div_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/dropout/div_grad/Neg" op: "Neg" input: "bert/encoder/layer_0/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/dropout/div_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/encoder/layer_0/attention/self/dropout/div_grad/Neg" input: "bert/encoder/layer_0/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/dropout/div_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/encoder/layer_0/attention/self/dropout/div_grad/RealDiv_1" input: "bert/encoder/layer_0/attention/self/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/dropout/div_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_0/attention/self/dropout/mul_grad/Mul" input: "gradients/bert/encoder/layer_0/attention/self/dropout/div_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/dropout/div_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_0/attention/self/dropout/div_grad/mul" input: "gradients/bert/encoder/layer_0/attention/self/dropout/div_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/attention/self/dropout/div_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/self/dropout/div_grad/Sum_1" input: "gradients/bert/encoder/layer_0/attention/self/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Reshape_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Reshape_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/self/transpose_2_grad/transpose" input: "gradients/bert/encoder/layer_0/attention/self/Reshape_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Softmax_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_0/attention/self/dropout/div_grad/Reshape" input: "bert/encoder/layer_0/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Softmax_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: -1 } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Softmax_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_0/attention/self/Softmax_grad/mul" input: "gradients/bert/encoder/layer_0/attention/self/Softmax_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Softmax_grad/sub" op: "Sub" input: "gradients/bert/encoder/layer_0/attention/self/dropout/div_grad/Reshape" input: "gradients/bert/encoder/layer_0/attention/self/Softmax_grad/Sum" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Softmax_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_0/attention/self/Softmax_grad/sub" input: "bert/encoder/layer_0/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/value/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_0/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_0/attention/self/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\001\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_0/attention/self/add_grad/Shape" input: "gradients/bert/encoder/layer_0/attention/self/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_0/attention/self/Softmax_grad/mul_1" input: "gradients/bert/encoder/layer_0/attention/self/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/attention/self/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/self/add_grad/Sum" input: "gradients/bert/encoder/layer_0/attention/self/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/add_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_0/attention/self/Softmax_grad/mul_1" input: "gradients/bert/encoder/layer_0/attention/self/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/attention/self/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/self/add_grad/Sum_1" input: "gradients/bert/encoder/layer_0/attention/self/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 1 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/value/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_0/attention/self/Reshape_2_grad/Reshape" input: "bert/encoder/layer_0/attention/self/value/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_0/attention/self/value/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/Reshape_1" input: "gradients/bert/encoder/layer_0/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_0/attention/self/Mul_grad/Shape" input: "gradients/bert/encoder/layer_0/attention/self/Mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_0/attention/self/add_grad/Reshape" input: "bert/encoder/layer_0/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_0/attention/self/Mul_grad/Mul" input: "gradients/bert/encoder/layer_0/attention/self/Mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/self/Mul_grad/Sum" input: "gradients/bert/encoder/layer_0/attention/self/Mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_0/attention/self/MatMul" input: "gradients/bert/encoder/layer_0/attention/self/add_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 128 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_0/attention/self/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_0/attention/self/Mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/self/Mul_grad/Sum_1" input: "gradients/bert/encoder/layer_0/attention/self/Mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/MatMul_grad/MatMul" op: "BatchMatMul" input: "gradients/bert/encoder/layer_0/attention/self/Mul_grad/Reshape" input: "bert/encoder/layer_0/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/attention/self/MatMul_grad/MatMul_1" op: "BatchMatMul" input: "gradients/bert/encoder/layer_0/attention/self/Mul_grad/Reshape" input: "bert/encoder/layer_0/attention/self/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 12 } dim { size: 128 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/attention/self/transpose_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_0/attention/self/transpose/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/transpose_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_0/attention/self/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_0/attention/self/transpose_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/transpose_1_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_0/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/transpose_1_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_0/attention/self/MatMul_grad/MatMul_1" input: "gradients/bert/encoder/layer_0/attention/self/transpose_1_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Reshape_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Reshape_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/self/transpose_grad/transpose" input: "gradients/bert/encoder/layer_0/attention/self/Reshape_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Reshape_1_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Reshape_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/self/transpose_1_grad/transpose" input: "gradients/bert/encoder/layer_0/attention/self/Reshape_1_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/query/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_0/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_0/attention/self/key/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_0/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_0/attention/self/query/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_0/attention/self/Reshape_grad/Reshape" input: "bert/encoder/layer_0/attention/self/query/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_0/attention/self/query/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/Reshape_1" input: "gradients/bert/encoder/layer_0/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/attention/self/key/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_0/attention/self/Reshape_1_grad/Reshape" input: "bert/encoder/layer_0/attention/self/key/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_0/attention/self/key/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/Reshape_1" input: "gradients/bert/encoder/layer_0/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_87" op: "AddN" input: "gradients/AddN_86" input: "gradients/bert/encoder/layer_0/attention/self/value/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_0/attention/self/query/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_0/attention/self/key/MatMul_grad/MatMul" attr { key: "N" value { i: 4 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/Reshape_1_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: " \000\000\000\200\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/Reshape_1_grad/Reshape" op: "Reshape" input: "gradients/AddN_87" input: "gradients/bert/encoder/Reshape_1_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/dropout/mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/Reshape_1_grad/Reshape" input: "bert/embeddings/dropout/Floor" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/dropout/mul_grad/Mul_1" op: "Mul" input: "gradients/bert/encoder/Reshape_1_grad/Reshape" input: "bert/embeddings/dropout/div" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/dropout/div_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: " \000\000\000\200\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/embeddings/dropout/div_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/embeddings/dropout/div_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/embeddings/dropout/div_grad/Shape" input: "gradients/bert/embeddings/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/embeddings/dropout/div_grad/RealDiv" op: "RealDiv" input: "gradients/bert/embeddings/dropout/mul_grad/Mul" input: "bert/embeddings/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/dropout/div_grad/Sum" op: "Sum" input: "gradients/bert/embeddings/dropout/div_grad/RealDiv" input: "gradients/bert/embeddings/dropout/div_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/embeddings/dropout/div_grad/Reshape" op: "Reshape" input: "gradients/bert/embeddings/dropout/div_grad/Sum" input: "gradients/bert/embeddings/dropout/div_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/dropout/div_grad/Neg" op: "Neg" input: "bert/embeddings/LayerNorm/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/dropout/div_grad/RealDiv_1" op: "RealDiv" input: "gradients/bert/embeddings/dropout/div_grad/Neg" input: "bert/embeddings/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/dropout/div_grad/RealDiv_2" op: "RealDiv" input: "gradients/bert/embeddings/dropout/div_grad/RealDiv_1" input: "bert/embeddings/dropout/keep_prob" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/dropout/div_grad/mul" op: "Mul" input: "gradients/bert/embeddings/dropout/mul_grad/Mul" input: "gradients/bert/embeddings/dropout/div_grad/RealDiv_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/dropout/div_grad/Sum_1" op: "Sum" input: "gradients/bert/embeddings/dropout/div_grad/mul" input: "gradients/bert/embeddings/dropout/div_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/embeddings/dropout/div_grad/Reshape_1" op: "Reshape" input: "gradients/bert/embeddings/dropout/div_grad/Sum_1" input: "gradients/bert/embeddings/dropout/div_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/bert/embeddings/dropout/div_grad/Reshape" input: "bert/embeddings/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/bert/embeddings/dropout/div_grad/Reshape" input: "bert/embeddings/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/batchnorm/sub_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/embeddings/LayerNorm/batchnorm/sub_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: " \000\000\000\200\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/embeddings/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/embeddings/LayerNorm/batchnorm/sub_grad/Shape" input: "gradients/bert/embeddings/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/bert/embeddings/dropout/div_grad/Reshape" input: "gradients/bert/embeddings/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/embeddings/LayerNorm/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/embeddings/LayerNorm/batchnorm/sub_grad/Sum" input: "gradients/bert/embeddings/LayerNorm/batchnorm/sub_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/batchnorm/sub_grad/Sum_1" op: "Sum" input: "gradients/bert/embeddings/dropout/div_grad/Reshape" input: "gradients/bert/embeddings/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/embeddings/LayerNorm/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/bert/embeddings/LayerNorm/batchnorm/sub_grad/Sum_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/batchnorm/sub_grad/Reshape_1" op: "Reshape" input: "gradients/bert/embeddings/LayerNorm/batchnorm/sub_grad/Neg" input: "gradients/bert/embeddings/LayerNorm/batchnorm/sub_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: " \000\000\000\200\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: " \000\000\000\200\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_2_grad/Shape" input: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/embeddings/LayerNorm/batchnorm/sub_grad/Reshape_1" input: "bert/embeddings/LayerNorm/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_2_grad/Mul" input: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_2_grad/Sum" input: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 1 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/embeddings/LayerNorm/moments/mean" input: "gradients/bert/embeddings/LayerNorm/batchnorm/sub_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_2_grad/Sum_1" op: "Sum" input: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_2_grad/Mul_1" input: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_2_grad/Sum_1" input: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_2_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "gradients/AddN_88" op: "AddN" input: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_2_grad/Reshape_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/LayerNorm/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: " \000\000\000\200\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_grad/Shape" input: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_88" input: "bert/embeddings/LayerNorm/gamma/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_grad/Mul" input: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_grad/Sum" input: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 1 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/embeddings/LayerNorm/batchnorm/Rsqrt" input: "gradients/AddN_88" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_grad/Mul_1" input: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_grad/Sum_1" input: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/embeddings/LayerNorm/batchnorm/Rsqrt" input: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 1 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/batchnorm/add_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: " \000\000\000\200\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/embeddings/LayerNorm/batchnorm/add_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/embeddings/LayerNorm/batchnorm/add_grad/Shape" input: "gradients/bert/embeddings/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/embeddings/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/embeddings/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/embeddings/LayerNorm/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/embeddings/LayerNorm/batchnorm/add_grad/Sum" input: "gradients/bert/embeddings/LayerNorm/batchnorm/add_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 1 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/batchnorm/add_grad/Sum_1" op: "Sum" input: "gradients/bert/embeddings/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/embeddings/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/embeddings/LayerNorm/batchnorm/add_grad/Reshape_1" op: "Reshape" input: "gradients/bert/embeddings/LayerNorm/batchnorm/add_grad/Sum_1" input: "gradients/bert/embeddings/LayerNorm/batchnorm/add_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/variance_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: " \000\000\000\200\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/variance_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 3 } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/variance_grad/add" op: "Add" input: "bert/embeddings/LayerNorm/moments/variance/reduction_indices" input: "gradients/bert/embeddings/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/variance_grad/mod" op: "FloorMod" input: "gradients/bert/embeddings/LayerNorm/moments/variance_grad/add" input: "gradients/bert/embeddings/LayerNorm/moments/variance_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/variance_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/variance_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/variance_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/variance_grad/range" op: "Range" input: "gradients/bert/embeddings/LayerNorm/moments/variance_grad/range/start" input: "gradients/bert/embeddings/LayerNorm/moments/variance_grad/Size" input: "gradients/bert/embeddings/LayerNorm/moments/variance_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/variance_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/variance_grad/Fill" op: "Fill" input: "gradients/bert/embeddings/LayerNorm/moments/variance_grad/Shape_1" input: "gradients/bert/embeddings/LayerNorm/moments/variance_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/variance_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/embeddings/LayerNorm/moments/variance_grad/range" input: "gradients/bert/embeddings/LayerNorm/moments/variance_grad/mod" input: "gradients/bert/embeddings/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/embeddings/LayerNorm/moments/variance_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/variance_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/variance_grad/Maximum" op: "Maximum" input: "gradients/bert/embeddings/LayerNorm/moments/variance_grad/DynamicStitch" input: "gradients/bert/embeddings/LayerNorm/moments/variance_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/variance_grad/floordiv" op: "FloorDiv" input: "gradients/bert/embeddings/LayerNorm/moments/variance_grad/Shape" input: "gradients/bert/embeddings/LayerNorm/moments/variance_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/LayerNorm/moments/variance_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/embeddings/LayerNorm/batchnorm/add_grad/Reshape" input: "gradients/bert/embeddings/LayerNorm/moments/variance_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 1 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/embeddings/LayerNorm/moments/variance_grad/Reshape" input: "gradients/bert/embeddings/LayerNorm/moments/variance_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/embeddings/LayerNorm/moments/variance_grad/Tile" input: "gradients/bert/embeddings/LayerNorm/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/SquaredDifference_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: " \000\000\000\200\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/SquaredDifference_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: " \000\000\000\200\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/embeddings/LayerNorm/moments/SquaredDifference_grad/Shape" input: "gradients/bert/embeddings/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/embeddings/LayerNorm/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/SquaredDifference_grad/mul" op: "Mul" input: "gradients/bert/embeddings/LayerNorm/moments/SquaredDifference_grad/scalar" input: "gradients/bert/embeddings/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/embeddings/add_1" input: "bert/embeddings/LayerNorm/moments/StopGradient" input: "^gradients/bert/embeddings/LayerNorm/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/embeddings/LayerNorm/moments/SquaredDifference_grad/mul" input: "gradients/bert/embeddings/LayerNorm/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/embeddings/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/embeddings/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/embeddings/LayerNorm/moments/SquaredDifference_grad/Sum" input: "gradients/bert/embeddings/LayerNorm/moments/SquaredDifference_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/SquaredDifference_grad/Sum_1" op: "Sum" input: "gradients/bert/embeddings/LayerNorm/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/embeddings/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/SquaredDifference_grad/Reshape_1" op: "Reshape" input: "gradients/bert/embeddings/LayerNorm/moments/SquaredDifference_grad/Sum_1" input: "gradients/bert/embeddings/LayerNorm/moments/SquaredDifference_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 1 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/embeddings/LayerNorm/moments/SquaredDifference_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 1 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/mean_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: " \000\000\000\200\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/mean_grad/Size" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 3 } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/mean_grad/add" op: "Add" input: "bert/embeddings/LayerNorm/moments/mean/reduction_indices" input: "gradients/bert/embeddings/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/mean_grad/mod" op: "FloorMod" input: "gradients/bert/embeddings/LayerNorm/moments/mean_grad/add" input: "gradients/bert/embeddings/LayerNorm/moments/mean_grad/Size" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/mean_grad/Shape_1" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/mean_grad/range/start" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/mean_grad/range/delta" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/mean_grad/range" op: "Range" input: "gradients/bert/embeddings/LayerNorm/moments/mean_grad/range/start" input: "gradients/bert/embeddings/LayerNorm/moments/mean_grad/Size" input: "gradients/bert/embeddings/LayerNorm/moments/mean_grad/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/mean_grad/Fill/value" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/mean_grad/Fill" op: "Fill" input: "gradients/bert/embeddings/LayerNorm/moments/mean_grad/Shape_1" input: "gradients/bert/embeddings/LayerNorm/moments/mean_grad/Fill/value" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/mean_grad/DynamicStitch" op: "DynamicStitch" input: "gradients/bert/embeddings/LayerNorm/moments/mean_grad/range" input: "gradients/bert/embeddings/LayerNorm/moments/mean_grad/mod" input: "gradients/bert/embeddings/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/embeddings/LayerNorm/moments/mean_grad/Fill" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/mean_grad/Maximum/y" op: "Const" attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/mean_grad/Maximum" op: "Maximum" input: "gradients/bert/embeddings/LayerNorm/moments/mean_grad/DynamicStitch" input: "gradients/bert/embeddings/LayerNorm/moments/mean_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/mean_grad/floordiv" op: "FloorDiv" input: "gradients/bert/embeddings/LayerNorm/moments/mean_grad/Shape" input: "gradients/bert/embeddings/LayerNorm/moments/mean_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/LayerNorm/moments/mean_grad/Shape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/embeddings/LayerNorm/moments/mean_grad/DynamicStitch" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 1 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/embeddings/LayerNorm/moments/mean_grad/Reshape" input: "gradients/bert/embeddings/LayerNorm/moments/mean_grad/floordiv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/embeddings/LayerNorm/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/embeddings/LayerNorm/moments/mean_grad/Tile" input: "gradients/bert/embeddings/LayerNorm/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "gradients/AddN_89" op: "AddN" input: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_1_grad/Mul" input: "gradients/bert/embeddings/LayerNorm/moments/SquaredDifference_grad/Reshape" input: "gradients/bert/embeddings/LayerNorm/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/LayerNorm/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/add_1_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: " \000\000\000\200\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/embeddings/add_1_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\001\000\000\000\200\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/embeddings/add_1_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/embeddings/add_1_grad/Shape" input: "gradients/bert/embeddings/add_1_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/embeddings/add_1_grad/Sum" op: "Sum" input: "gradients/AddN_89" input: "gradients/bert/embeddings/add_1_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/embeddings/add_1_grad/Reshape" op: "Reshape" input: "gradients/bert/embeddings/add_1_grad/Sum" input: "gradients/bert/embeddings/add_1_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/add_1_grad/Sum_1" op: "Sum" input: "gradients/AddN_89" input: "gradients/bert/embeddings/add_1_grad/BroadcastGradientArgs:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 128 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/embeddings/add_1_grad/Reshape_1" op: "Reshape" input: "gradients/bert/embeddings/add_1_grad/Sum_1" input: "gradients/bert/embeddings/add_1_grad/Shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 128 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/Reshape_3_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\200\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/embeddings/Reshape_3_grad/Reshape" op: "Reshape" input: "gradients/bert/embeddings/add_1_grad/Reshape_1" input: "gradients/bert/embeddings/Reshape_3_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 128 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/Reshape_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: " \000\000\000\200\000\000\000\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/embeddings/Reshape_grad/Reshape" op: "Reshape" input: "gradients/bert/embeddings/add_1_grad/Reshape" input: "gradients/bert/embeddings/Reshape_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 32 } dim { size: 128 } dim { size: 1 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/Reshape_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\020\000\000\000\003\000\000" } } } } node { name: "gradients/bert/embeddings/Reshape_2_grad/Reshape" op: "Reshape" input: "gradients/bert/embeddings/add_1_grad/Reshape" input: "gradients/bert/embeddings/Reshape_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/Slice_grad/Rank" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/embeddings/Slice_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\200\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/embeddings/Slice_grad/stack/1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/embeddings/Slice_grad/stack" op: "Pack" input: "gradients/bert/embeddings/Slice_grad/Rank" input: "gradients/bert/embeddings/Slice_grad/stack/1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "axis" value { i: 0 } } } node { name: "gradients/bert/embeddings/Slice_grad/Reshape" op: "Reshape" input: "bert/embeddings/Slice/begin" input: "gradients/bert/embeddings/Slice_grad/stack" attr { key: "T" value { type: DT_INT32 } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 1 } } } } } } node { name: "gradients/bert/embeddings/Slice_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\002\000\000\000\003\000\000" } } } } node { name: "gradients/bert/embeddings/Slice_grad/sub" op: "Sub" input: "gradients/bert/embeddings/Slice_grad/Shape_1" input: "gradients/bert/embeddings/Slice_grad/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/embeddings/Slice_grad/sub_1" op: "Sub" input: "gradients/bert/embeddings/Slice_grad/sub" input: "bert/embeddings/Slice/begin" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/embeddings/Slice_grad/Reshape_1" op: "Reshape" input: "gradients/bert/embeddings/Slice_grad/sub_1" input: "gradients/bert/embeddings/Slice_grad/stack" attr { key: "T" value { type: DT_INT32 } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 1 } } } } } } node { name: "gradients/bert/embeddings/Slice_grad/concat/axis" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/embeddings/Slice_grad/concat" op: "ConcatV2" input: "gradients/bert/embeddings/Slice_grad/Reshape" input: "gradients/bert/embeddings/Slice_grad/Reshape_1" input: "gradients/bert/embeddings/Slice_grad/concat/axis" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 2 } } } } } } node { name: "gradients/bert/embeddings/Slice_grad/Pad" op: "Pad" input: "gradients/bert/embeddings/Reshape_3_grad/Reshape" input: "gradients/bert/embeddings/Slice_grad/concat" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tpaddings" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/embeddings/Reshape_2_grad/Reshape" input: "bert/embeddings/token_type_embeddings/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 2 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/embeddings/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/embeddings/one_hot" input: "gradients/bert/embeddings/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/embeddings/embedding_lookup_grad/Shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT64 } } attr { key: "value" value { tensor { dtype: DT_INT64 tensor_shape { dim { size: 2 } } tensor_content: "Dq\000\000\000\000\000\000\000\003\000\000\000\000\000\000" } } } } node { name: "gradients/bert/embeddings/embedding_lookup_grad/ToInt32" op: "Cast" input: "gradients/bert/embeddings/embedding_lookup_grad/Shape" attr { key: "DstT" value { type: DT_INT32 } } attr { key: "SrcT" value { type: DT_INT64 } } attr { key: "Truncate" value { b: false } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/embeddings/embedding_lookup_grad/Size" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 4096 } } } } node { name: "gradients/bert/embeddings/embedding_lookup_grad/ExpandDims/dim" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/embeddings/embedding_lookup_grad/ExpandDims" op: "ExpandDims" input: "gradients/bert/embeddings/embedding_lookup_grad/Size" input: "gradients/bert/embeddings/embedding_lookup_grad/ExpandDims/dim" attr { key: "T" value { type: DT_INT32 } } attr { key: "Tdim" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/embeddings/embedding_lookup_grad/strided_slice/stack" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/embeddings/embedding_lookup_grad/strided_slice/stack_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/embeddings/embedding_lookup_grad/strided_slice/stack_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/embeddings/embedding_lookup_grad/strided_slice" op: "StridedSlice" input: "gradients/bert/embeddings/embedding_lookup_grad/ToInt32" input: "gradients/bert/embeddings/embedding_lookup_grad/strided_slice/stack" input: "gradients/bert/embeddings/embedding_lookup_grad/strided_slice/stack_1" input: "gradients/bert/embeddings/embedding_lookup_grad/strided_slice/stack_2" attr { key: "Index" value { type: DT_INT32 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "begin_mask" value { i: 0 } } attr { key: "ellipsis_mask" value { i: 0 } } attr { key: "end_mask" value { i: 1 } } attr { key: "new_axis_mask" value { i: 0 } } attr { key: "shrink_axis_mask" value { i: 0 } } } node { name: "gradients/bert/embeddings/embedding_lookup_grad/concat/axis" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/embeddings/embedding_lookup_grad/concat" op: "ConcatV2" input: "gradients/bert/embeddings/embedding_lookup_grad/ExpandDims" input: "gradients/bert/embeddings/embedding_lookup_grad/strided_slice" input: "gradients/bert/embeddings/embedding_lookup_grad/concat/axis" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/embeddings/embedding_lookup_grad/Reshape" op: "Reshape" input: "gradients/bert/embeddings/Reshape_grad/Reshape" input: "gradients/bert/embeddings/embedding_lookup_grad/concat" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/embedding_lookup_grad/Reshape_1" op: "Reshape" input: "bert/embeddings/ExpandDims" input: "gradients/bert/embeddings/embedding_lookup_grad/ExpandDims" attr { key: "T" value { type: DT_INT32 } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4096 } } } } } } node { name: "gradients/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "Dq\000\000\000\003\000\000" } } } } node { name: "gradients/strided_slice/stack" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/strided_slice/stack_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/strided_slice/stack_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/strided_slice" op: "StridedSlice" input: "gradients/Shape_1" input: "gradients/strided_slice/stack" input: "gradients/strided_slice/stack_1" input: "gradients/strided_slice/stack_2" attr { key: "Index" value { type: DT_INT32 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "begin_mask" value { i: 0 } } attr { key: "ellipsis_mask" value { i: 0 } } attr { key: "end_mask" value { i: 0 } } attr { key: "new_axis_mask" value { i: 0 } } attr { key: "shrink_axis_mask" value { i: 1 } } } node { name: "gradients/range/start" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/range/delta" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/range" op: "Range" input: "gradients/range/start" input: "gradients/strided_slice" input: "gradients/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } } } } } } node { name: "gradients/concat/axis" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/concat" op: "ConcatV2" input: "gradients/cls/predictions/MatMul_grad/MatMul_1" input: "gradients/bert/embeddings/embedding_lookup_grad/Reshape" input: "gradients/concat/axis" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 33092 } dim { size: 768 } } } } } } node { name: "gradients/concat_1/axis" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/concat_1" op: "ConcatV2" input: "gradients/range" input: "gradients/bert/embeddings/embedding_lookup_grad/Reshape_1" input: "gradients/concat_1/axis" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 33092 } } } } } } node { name: "global_norm/L2Loss" op: "L2Loss" input: "gradients/concat" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/concat" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_1" op: "L2Loss" input: "gradients/bert/embeddings/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_2" op: "L2Loss" input: "gradients/bert/embeddings/Slice_grad/Pad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/Slice_grad/Pad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_3" op: "L2Loss" input: "gradients/bert/embeddings/LayerNorm/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_4" op: "L2Loss" input: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_5" op: "L2Loss" input: "gradients/bert/encoder/layer_0/attention/self/query/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_6" op: "L2Loss" input: "gradients/bert/encoder/layer_0/attention/self/query/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_7" op: "L2Loss" input: "gradients/bert/encoder/layer_0/attention/self/key/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_8" op: "L2Loss" input: "gradients/bert/encoder/layer_0/attention/self/key/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_9" op: "L2Loss" input: "gradients/bert/encoder/layer_0/attention/self/value/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_10" op: "L2Loss" input: "gradients/bert/encoder/layer_0/attention/self/value/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_11" op: "L2Loss" input: "gradients/bert/encoder/layer_0/attention/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_12" op: "L2Loss" input: "gradients/bert/encoder/layer_0/attention/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_13" op: "L2Loss" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_14" op: "L2Loss" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_15" op: "L2Loss" input: "gradients/bert/encoder/layer_0/intermediate/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_16" op: "L2Loss" input: "gradients/bert/encoder/layer_0/intermediate/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_17" op: "L2Loss" input: "gradients/bert/encoder/layer_0/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_18" op: "L2Loss" input: "gradients/bert/encoder/layer_0/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_19" op: "L2Loss" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_20" op: "L2Loss" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_21" op: "L2Loss" input: "gradients/bert/encoder/layer_1/attention/self/query/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_22" op: "L2Loss" input: "gradients/bert/encoder/layer_1/attention/self/query/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_23" op: "L2Loss" input: "gradients/bert/encoder/layer_1/attention/self/key/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_24" op: "L2Loss" input: "gradients/bert/encoder/layer_1/attention/self/key/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_25" op: "L2Loss" input: "gradients/bert/encoder/layer_1/attention/self/value/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_26" op: "L2Loss" input: "gradients/bert/encoder/layer_1/attention/self/value/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_27" op: "L2Loss" input: "gradients/bert/encoder/layer_1/attention/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_28" op: "L2Loss" input: "gradients/bert/encoder/layer_1/attention/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_29" op: "L2Loss" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_30" op: "L2Loss" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_31" op: "L2Loss" input: "gradients/bert/encoder/layer_1/intermediate/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_32" op: "L2Loss" input: "gradients/bert/encoder/layer_1/intermediate/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_33" op: "L2Loss" input: "gradients/bert/encoder/layer_1/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_34" op: "L2Loss" input: "gradients/bert/encoder/layer_1/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_35" op: "L2Loss" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_36" op: "L2Loss" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_37" op: "L2Loss" input: "gradients/bert/encoder/layer_2/attention/self/query/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_38" op: "L2Loss" input: "gradients/bert/encoder/layer_2/attention/self/query/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_39" op: "L2Loss" input: "gradients/bert/encoder/layer_2/attention/self/key/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_40" op: "L2Loss" input: "gradients/bert/encoder/layer_2/attention/self/key/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_41" op: "L2Loss" input: "gradients/bert/encoder/layer_2/attention/self/value/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_42" op: "L2Loss" input: "gradients/bert/encoder/layer_2/attention/self/value/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_43" op: "L2Loss" input: "gradients/bert/encoder/layer_2/attention/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_44" op: "L2Loss" input: "gradients/bert/encoder/layer_2/attention/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_45" op: "L2Loss" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_46" op: "L2Loss" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_47" op: "L2Loss" input: "gradients/bert/encoder/layer_2/intermediate/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_48" op: "L2Loss" input: "gradients/bert/encoder/layer_2/intermediate/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_49" op: "L2Loss" input: "gradients/bert/encoder/layer_2/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_50" op: "L2Loss" input: "gradients/bert/encoder/layer_2/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_51" op: "L2Loss" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_52" op: "L2Loss" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_53" op: "L2Loss" input: "gradients/bert/encoder/layer_3/attention/self/query/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_54" op: "L2Loss" input: "gradients/bert/encoder/layer_3/attention/self/query/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_55" op: "L2Loss" input: "gradients/bert/encoder/layer_3/attention/self/key/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_56" op: "L2Loss" input: "gradients/bert/encoder/layer_3/attention/self/key/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_57" op: "L2Loss" input: "gradients/bert/encoder/layer_3/attention/self/value/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_58" op: "L2Loss" input: "gradients/bert/encoder/layer_3/attention/self/value/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_59" op: "L2Loss" input: "gradients/bert/encoder/layer_3/attention/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_60" op: "L2Loss" input: "gradients/bert/encoder/layer_3/attention/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_61" op: "L2Loss" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_62" op: "L2Loss" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_63" op: "L2Loss" input: "gradients/bert/encoder/layer_3/intermediate/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_64" op: "L2Loss" input: "gradients/bert/encoder/layer_3/intermediate/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_65" op: "L2Loss" input: "gradients/bert/encoder/layer_3/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_66" op: "L2Loss" input: "gradients/bert/encoder/layer_3/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_67" op: "L2Loss" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_68" op: "L2Loss" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_69" op: "L2Loss" input: "gradients/bert/encoder/layer_4/attention/self/query/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_70" op: "L2Loss" input: "gradients/bert/encoder/layer_4/attention/self/query/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_71" op: "L2Loss" input: "gradients/bert/encoder/layer_4/attention/self/key/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_72" op: "L2Loss" input: "gradients/bert/encoder/layer_4/attention/self/key/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_73" op: "L2Loss" input: "gradients/bert/encoder/layer_4/attention/self/value/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_74" op: "L2Loss" input: "gradients/bert/encoder/layer_4/attention/self/value/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_75" op: "L2Loss" input: "gradients/bert/encoder/layer_4/attention/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_76" op: "L2Loss" input: "gradients/bert/encoder/layer_4/attention/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_77" op: "L2Loss" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_78" op: "L2Loss" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_79" op: "L2Loss" input: "gradients/bert/encoder/layer_4/intermediate/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_80" op: "L2Loss" input: "gradients/bert/encoder/layer_4/intermediate/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_81" op: "L2Loss" input: "gradients/bert/encoder/layer_4/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_82" op: "L2Loss" input: "gradients/bert/encoder/layer_4/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_83" op: "L2Loss" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_84" op: "L2Loss" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_85" op: "L2Loss" input: "gradients/bert/encoder/layer_5/attention/self/query/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_86" op: "L2Loss" input: "gradients/bert/encoder/layer_5/attention/self/query/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_87" op: "L2Loss" input: "gradients/bert/encoder/layer_5/attention/self/key/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_88" op: "L2Loss" input: "gradients/bert/encoder/layer_5/attention/self/key/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_89" op: "L2Loss" input: "gradients/bert/encoder/layer_5/attention/self/value/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_90" op: "L2Loss" input: "gradients/bert/encoder/layer_5/attention/self/value/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_91" op: "L2Loss" input: "gradients/bert/encoder/layer_5/attention/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_92" op: "L2Loss" input: "gradients/bert/encoder/layer_5/attention/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_93" op: "L2Loss" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_94" op: "L2Loss" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_95" op: "L2Loss" input: "gradients/bert/encoder/layer_5/intermediate/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_96" op: "L2Loss" input: "gradients/bert/encoder/layer_5/intermediate/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_97" op: "L2Loss" input: "gradients/bert/encoder/layer_5/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_98" op: "L2Loss" input: "gradients/bert/encoder/layer_5/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_99" op: "L2Loss" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_100" op: "L2Loss" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_101" op: "L2Loss" input: "gradients/bert/encoder/layer_6/attention/self/query/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_102" op: "L2Loss" input: "gradients/bert/encoder/layer_6/attention/self/query/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_103" op: "L2Loss" input: "gradients/bert/encoder/layer_6/attention/self/key/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_104" op: "L2Loss" input: "gradients/bert/encoder/layer_6/attention/self/key/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_105" op: "L2Loss" input: "gradients/bert/encoder/layer_6/attention/self/value/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_106" op: "L2Loss" input: "gradients/bert/encoder/layer_6/attention/self/value/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_107" op: "L2Loss" input: "gradients/bert/encoder/layer_6/attention/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_108" op: "L2Loss" input: "gradients/bert/encoder/layer_6/attention/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_109" op: "L2Loss" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_110" op: "L2Loss" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_111" op: "L2Loss" input: "gradients/bert/encoder/layer_6/intermediate/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_112" op: "L2Loss" input: "gradients/bert/encoder/layer_6/intermediate/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_113" op: "L2Loss" input: "gradients/bert/encoder/layer_6/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_114" op: "L2Loss" input: "gradients/bert/encoder/layer_6/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_115" op: "L2Loss" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_116" op: "L2Loss" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_117" op: "L2Loss" input: "gradients/bert/encoder/layer_7/attention/self/query/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_118" op: "L2Loss" input: "gradients/bert/encoder/layer_7/attention/self/query/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_119" op: "L2Loss" input: "gradients/bert/encoder/layer_7/attention/self/key/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_120" op: "L2Loss" input: "gradients/bert/encoder/layer_7/attention/self/key/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_121" op: "L2Loss" input: "gradients/bert/encoder/layer_7/attention/self/value/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_122" op: "L2Loss" input: "gradients/bert/encoder/layer_7/attention/self/value/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_123" op: "L2Loss" input: "gradients/bert/encoder/layer_7/attention/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_124" op: "L2Loss" input: "gradients/bert/encoder/layer_7/attention/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_125" op: "L2Loss" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_126" op: "L2Loss" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_127" op: "L2Loss" input: "gradients/bert/encoder/layer_7/intermediate/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_128" op: "L2Loss" input: "gradients/bert/encoder/layer_7/intermediate/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_129" op: "L2Loss" input: "gradients/bert/encoder/layer_7/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_130" op: "L2Loss" input: "gradients/bert/encoder/layer_7/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_131" op: "L2Loss" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_132" op: "L2Loss" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_133" op: "L2Loss" input: "gradients/bert/encoder/layer_8/attention/self/query/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_134" op: "L2Loss" input: "gradients/bert/encoder/layer_8/attention/self/query/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_135" op: "L2Loss" input: "gradients/bert/encoder/layer_8/attention/self/key/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_136" op: "L2Loss" input: "gradients/bert/encoder/layer_8/attention/self/key/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_137" op: "L2Loss" input: "gradients/bert/encoder/layer_8/attention/self/value/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_138" op: "L2Loss" input: "gradients/bert/encoder/layer_8/attention/self/value/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_139" op: "L2Loss" input: "gradients/bert/encoder/layer_8/attention/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_140" op: "L2Loss" input: "gradients/bert/encoder/layer_8/attention/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_141" op: "L2Loss" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_142" op: "L2Loss" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_143" op: "L2Loss" input: "gradients/bert/encoder/layer_8/intermediate/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_144" op: "L2Loss" input: "gradients/bert/encoder/layer_8/intermediate/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_145" op: "L2Loss" input: "gradients/bert/encoder/layer_8/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_146" op: "L2Loss" input: "gradients/bert/encoder/layer_8/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_147" op: "L2Loss" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_148" op: "L2Loss" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_149" op: "L2Loss" input: "gradients/bert/encoder/layer_9/attention/self/query/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_150" op: "L2Loss" input: "gradients/bert/encoder/layer_9/attention/self/query/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_151" op: "L2Loss" input: "gradients/bert/encoder/layer_9/attention/self/key/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_152" op: "L2Loss" input: "gradients/bert/encoder/layer_9/attention/self/key/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_153" op: "L2Loss" input: "gradients/bert/encoder/layer_9/attention/self/value/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_154" op: "L2Loss" input: "gradients/bert/encoder/layer_9/attention/self/value/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_155" op: "L2Loss" input: "gradients/bert/encoder/layer_9/attention/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_156" op: "L2Loss" input: "gradients/bert/encoder/layer_9/attention/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_157" op: "L2Loss" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_158" op: "L2Loss" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_159" op: "L2Loss" input: "gradients/bert/encoder/layer_9/intermediate/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_160" op: "L2Loss" input: "gradients/bert/encoder/layer_9/intermediate/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_161" op: "L2Loss" input: "gradients/bert/encoder/layer_9/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_162" op: "L2Loss" input: "gradients/bert/encoder/layer_9/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_163" op: "L2Loss" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_164" op: "L2Loss" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_165" op: "L2Loss" input: "gradients/bert/encoder/layer_10/attention/self/query/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_166" op: "L2Loss" input: "gradients/bert/encoder/layer_10/attention/self/query/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_167" op: "L2Loss" input: "gradients/bert/encoder/layer_10/attention/self/key/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_168" op: "L2Loss" input: "gradients/bert/encoder/layer_10/attention/self/key/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_169" op: "L2Loss" input: "gradients/bert/encoder/layer_10/attention/self/value/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_170" op: "L2Loss" input: "gradients/bert/encoder/layer_10/attention/self/value/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_171" op: "L2Loss" input: "gradients/bert/encoder/layer_10/attention/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_172" op: "L2Loss" input: "gradients/bert/encoder/layer_10/attention/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_173" op: "L2Loss" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_174" op: "L2Loss" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_175" op: "L2Loss" input: "gradients/bert/encoder/layer_10/intermediate/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_176" op: "L2Loss" input: "gradients/bert/encoder/layer_10/intermediate/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_177" op: "L2Loss" input: "gradients/bert/encoder/layer_10/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_178" op: "L2Loss" input: "gradients/bert/encoder/layer_10/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_179" op: "L2Loss" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_180" op: "L2Loss" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_181" op: "L2Loss" input: "gradients/bert/encoder/layer_11/attention/self/query/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_182" op: "L2Loss" input: "gradients/bert/encoder/layer_11/attention/self/query/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_183" op: "L2Loss" input: "gradients/bert/encoder/layer_11/attention/self/key/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_184" op: "L2Loss" input: "gradients/bert/encoder/layer_11/attention/self/key/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_185" op: "L2Loss" input: "gradients/bert/encoder/layer_11/attention/self/value/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_186" op: "L2Loss" input: "gradients/bert/encoder/layer_11/attention/self/value/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_187" op: "L2Loss" input: "gradients/bert/encoder/layer_11/attention/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_188" op: "L2Loss" input: "gradients/bert/encoder/layer_11/attention/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_189" op: "L2Loss" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_190" op: "L2Loss" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_191" op: "L2Loss" input: "gradients/bert/encoder/layer_11/intermediate/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_192" op: "L2Loss" input: "gradients/bert/encoder/layer_11/intermediate/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_193" op: "L2Loss" input: "gradients/bert/encoder/layer_11/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_194" op: "L2Loss" input: "gradients/bert/encoder/layer_11/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_195" op: "L2Loss" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_196" op: "L2Loss" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_197" op: "L2Loss" input: "gradients/bert/pooler/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/pooler/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_198" op: "L2Loss" input: "gradients/bert/pooler/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/pooler/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_199" op: "L2Loss" input: "gradients/cls/predictions/transform/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/transform/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_200" op: "L2Loss" input: "gradients/cls/predictions/transform/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/transform/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_201" op: "L2Loss" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/transform/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_202" op: "L2Loss" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_203" op: "L2Loss" input: "gradients/cls/predictions/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_204" op: "L2Loss" input: "gradients/cls/seq_relationship/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/cls/seq_relationship/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_205" op: "L2Loss" input: "gradients/cls/seq_relationship/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/cls/seq_relationship/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/stack" op: "Pack" input: "global_norm/L2Loss" input: "global_norm/L2Loss_1" input: "global_norm/L2Loss_2" input: "global_norm/L2Loss_3" input: "global_norm/L2Loss_4" input: "global_norm/L2Loss_5" input: "global_norm/L2Loss_6" input: "global_norm/L2Loss_7" input: "global_norm/L2Loss_8" input: "global_norm/L2Loss_9" input: "global_norm/L2Loss_10" input: "global_norm/L2Loss_11" input: "global_norm/L2Loss_12" input: "global_norm/L2Loss_13" input: "global_norm/L2Loss_14" input: "global_norm/L2Loss_15" input: "global_norm/L2Loss_16" input: "global_norm/L2Loss_17" input: "global_norm/L2Loss_18" input: "global_norm/L2Loss_19" input: "global_norm/L2Loss_20" input: "global_norm/L2Loss_21" input: "global_norm/L2Loss_22" input: "global_norm/L2Loss_23" input: "global_norm/L2Loss_24" input: "global_norm/L2Loss_25" input: "global_norm/L2Loss_26" input: "global_norm/L2Loss_27" input: "global_norm/L2Loss_28" input: "global_norm/L2Loss_29" input: "global_norm/L2Loss_30" input: "global_norm/L2Loss_31" input: "global_norm/L2Loss_32" input: "global_norm/L2Loss_33" input: "global_norm/L2Loss_34" input: "global_norm/L2Loss_35" input: "global_norm/L2Loss_36" input: "global_norm/L2Loss_37" input: "global_norm/L2Loss_38" input: "global_norm/L2Loss_39" input: "global_norm/L2Loss_40" input: "global_norm/L2Loss_41" input: "global_norm/L2Loss_42" input: "global_norm/L2Loss_43" input: "global_norm/L2Loss_44" input: "global_norm/L2Loss_45" input: "global_norm/L2Loss_46" input: "global_norm/L2Loss_47" input: "global_norm/L2Loss_48" input: "global_norm/L2Loss_49" input: "global_norm/L2Loss_50" input: "global_norm/L2Loss_51" input: "global_norm/L2Loss_52" input: "global_norm/L2Loss_53" input: "global_norm/L2Loss_54" input: "global_norm/L2Loss_55" input: "global_norm/L2Loss_56" input: "global_norm/L2Loss_57" input: "global_norm/L2Loss_58" input: "global_norm/L2Loss_59" input: "global_norm/L2Loss_60" input: "global_norm/L2Loss_61" input: "global_norm/L2Loss_62" input: "global_norm/L2Loss_63" input: "global_norm/L2Loss_64" input: "global_norm/L2Loss_65" input: "global_norm/L2Loss_66" input: "global_norm/L2Loss_67" input: "global_norm/L2Loss_68" input: "global_norm/L2Loss_69" input: "global_norm/L2Loss_70" input: "global_norm/L2Loss_71" input: "global_norm/L2Loss_72" input: "global_norm/L2Loss_73" input: "global_norm/L2Loss_74" input: "global_norm/L2Loss_75" input: "global_norm/L2Loss_76" input: "global_norm/L2Loss_77" input: "global_norm/L2Loss_78" input: "global_norm/L2Loss_79" input: "global_norm/L2Loss_80" input: "global_norm/L2Loss_81" input: "global_norm/L2Loss_82" input: "global_norm/L2Loss_83" input: "global_norm/L2Loss_84" input: "global_norm/L2Loss_85" input: "global_norm/L2Loss_86" input: "global_norm/L2Loss_87" input: "global_norm/L2Loss_88" input: "global_norm/L2Loss_89" input: "global_norm/L2Loss_90" input: "global_norm/L2Loss_91" input: "global_norm/L2Loss_92" input: "global_norm/L2Loss_93" input: "global_norm/L2Loss_94" input: "global_norm/L2Loss_95" input: "global_norm/L2Loss_96" input: "global_norm/L2Loss_97" input: "global_norm/L2Loss_98" input: "global_norm/L2Loss_99" input: "global_norm/L2Loss_100" input: "global_norm/L2Loss_101" input: "global_norm/L2Loss_102" input: "global_norm/L2Loss_103" input: "global_norm/L2Loss_104" input: "global_norm/L2Loss_105" input: "global_norm/L2Loss_106" input: "global_norm/L2Loss_107" input: "global_norm/L2Loss_108" input: "global_norm/L2Loss_109" input: "global_norm/L2Loss_110" input: "global_norm/L2Loss_111" input: "global_norm/L2Loss_112" input: "global_norm/L2Loss_113" input: "global_norm/L2Loss_114" input: "global_norm/L2Loss_115" input: "global_norm/L2Loss_116" input: "global_norm/L2Loss_117" input: "global_norm/L2Loss_118" input: "global_norm/L2Loss_119" input: "global_norm/L2Loss_120" input: "global_norm/L2Loss_121" input: "global_norm/L2Loss_122" input: "global_norm/L2Loss_123" input: "global_norm/L2Loss_124" input: "global_norm/L2Loss_125" input: "global_norm/L2Loss_126" input: "global_norm/L2Loss_127" input: "global_norm/L2Loss_128" input: "global_norm/L2Loss_129" input: "global_norm/L2Loss_130" input: "global_norm/L2Loss_131" input: "global_norm/L2Loss_132" input: "global_norm/L2Loss_133" input: "global_norm/L2Loss_134" input: "global_norm/L2Loss_135" input: "global_norm/L2Loss_136" input: "global_norm/L2Loss_137" input: "global_norm/L2Loss_138" input: "global_norm/L2Loss_139" input: "global_norm/L2Loss_140" input: "global_norm/L2Loss_141" input: "global_norm/L2Loss_142" input: "global_norm/L2Loss_143" input: "global_norm/L2Loss_144" input: "global_norm/L2Loss_145" input: "global_norm/L2Loss_146" input: "global_norm/L2Loss_147" input: "global_norm/L2Loss_148" input: "global_norm/L2Loss_149" input: "global_norm/L2Loss_150" input: "global_norm/L2Loss_151" input: "global_norm/L2Loss_152" input: "global_norm/L2Loss_153" input: "global_norm/L2Loss_154" input: "global_norm/L2Loss_155" input: "global_norm/L2Loss_156" input: "global_norm/L2Loss_157" input: "global_norm/L2Loss_158" input: "global_norm/L2Loss_159" input: "global_norm/L2Loss_160" input: "global_norm/L2Loss_161" input: "global_norm/L2Loss_162" input: "global_norm/L2Loss_163" input: "global_norm/L2Loss_164" input: "global_norm/L2Loss_165" input: "global_norm/L2Loss_166" input: "global_norm/L2Loss_167" input: "global_norm/L2Loss_168" input: "global_norm/L2Loss_169" input: "global_norm/L2Loss_170" input: "global_norm/L2Loss_171" input: "global_norm/L2Loss_172" input: "global_norm/L2Loss_173" input: "global_norm/L2Loss_174" input: "global_norm/L2Loss_175" input: "global_norm/L2Loss_176" input: "global_norm/L2Loss_177" input: "global_norm/L2Loss_178" input: "global_norm/L2Loss_179" input: "global_norm/L2Loss_180" input: "global_norm/L2Loss_181" input: "global_norm/L2Loss_182" input: "global_norm/L2Loss_183" input: "global_norm/L2Loss_184" input: "global_norm/L2Loss_185" input: "global_norm/L2Loss_186" input: "global_norm/L2Loss_187" input: "global_norm/L2Loss_188" input: "global_norm/L2Loss_189" input: "global_norm/L2Loss_190" input: "global_norm/L2Loss_191" input: "global_norm/L2Loss_192" input: "global_norm/L2Loss_193" input: "global_norm/L2Loss_194" input: "global_norm/L2Loss_195" input: "global_norm/L2Loss_196" input: "global_norm/L2Loss_197" input: "global_norm/L2Loss_198" input: "global_norm/L2Loss_199" input: "global_norm/L2Loss_200" input: "global_norm/L2Loss_201" input: "global_norm/L2Loss_202" input: "global_norm/L2Loss_203" input: "global_norm/L2Loss_204" input: "global_norm/L2Loss_205" attr { key: "N" value { i: 206 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 206 } } } } } attr { key: "axis" value { i: 0 } } } node { name: "global_norm/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "global_norm/Sum" op: "Sum" input: "global_norm/stack" input: "global_norm/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "global_norm/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "global_norm/mul" op: "Mul" input: "global_norm/Sum" input: "global_norm/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/global_norm" op: "Sqrt" input: "global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "VerifyFinite/CheckNumerics" op: "CheckNumerics" input: "global_norm/global_norm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@global_norm/global_norm" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "message" value { s: "Found Inf or NaN global norm." } } } node { name: "VerifyFinite/control_dependency" op: "Identity" input: "global_norm/global_norm" input: "^VerifyFinite/CheckNumerics" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@global_norm/global_norm" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "clip_by_global_norm/truediv/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "clip_by_global_norm/truediv" op: "RealDiv" input: "clip_by_global_norm/truediv/x" input: "VerifyFinite/control_dependency" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "clip_by_global_norm/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "clip_by_global_norm/truediv_1/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "clip_by_global_norm/truediv_1" op: "RealDiv" input: "clip_by_global_norm/Const" input: "clip_by_global_norm/truediv_1/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "clip_by_global_norm/Minimum" op: "Minimum" input: "clip_by_global_norm/truediv" input: "clip_by_global_norm/truediv_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "clip_by_global_norm/mul/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "clip_by_global_norm/mul" op: "Mul" input: "clip_by_global_norm/mul/x" input: "clip_by_global_norm/Minimum" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "clip_by_global_norm/mul_1" op: "Mul" input: "gradients/concat" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/concat" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 33092 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_0" op: "Identity" input: "clip_by_global_norm/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/concat" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 33092 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_2" op: "Mul" input: "gradients/bert/embeddings/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_1" op: "Identity" input: "clip_by_global_norm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_3" op: "Mul" input: "gradients/bert/embeddings/Slice_grad/Pad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/Slice_grad/Pad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_2" op: "Identity" input: "clip_by_global_norm/mul_3" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/Slice_grad/Pad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_4" op: "Mul" input: "gradients/bert/embeddings/LayerNorm/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_3" op: "Identity" input: "clip_by_global_norm/mul_4" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_5" op: "Mul" input: "gradients/bert/embeddings/LayerNorm/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_4" op: "Identity" input: "clip_by_global_norm/mul_5" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_6" op: "Mul" input: "gradients/bert/encoder/layer_0/attention/self/query/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_5" op: "Identity" input: "clip_by_global_norm/mul_6" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_7" op: "Mul" input: "gradients/bert/encoder/layer_0/attention/self/query/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_6" op: "Identity" input: "clip_by_global_norm/mul_7" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_8" op: "Mul" input: "gradients/bert/encoder/layer_0/attention/self/key/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_7" op: "Identity" input: "clip_by_global_norm/mul_8" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_9" op: "Mul" input: "gradients/bert/encoder/layer_0/attention/self/key/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_8" op: "Identity" input: "clip_by_global_norm/mul_9" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_10" op: "Mul" input: "gradients/bert/encoder/layer_0/attention/self/value/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_9" op: "Identity" input: "clip_by_global_norm/mul_10" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_11" op: "Mul" input: "gradients/bert/encoder/layer_0/attention/self/value/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_10" op: "Identity" input: "clip_by_global_norm/mul_11" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_12" op: "Mul" input: "gradients/bert/encoder/layer_0/attention/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_11" op: "Identity" input: "clip_by_global_norm/mul_12" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_13" op: "Mul" input: "gradients/bert/encoder/layer_0/attention/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_12" op: "Identity" input: "clip_by_global_norm/mul_13" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_14" op: "Mul" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_13" op: "Identity" input: "clip_by_global_norm/mul_14" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_15" op: "Mul" input: "gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_14" op: "Identity" input: "clip_by_global_norm/mul_15" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_16" op: "Mul" input: "gradients/bert/encoder/layer_0/intermediate/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_15" op: "Identity" input: "clip_by_global_norm/mul_16" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_17" op: "Mul" input: "gradients/bert/encoder/layer_0/intermediate/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_16" op: "Identity" input: "clip_by_global_norm/mul_17" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_18" op: "Mul" input: "gradients/bert/encoder/layer_0/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_17" op: "Identity" input: "clip_by_global_norm/mul_18" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_19" op: "Mul" input: "gradients/bert/encoder/layer_0/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_18" op: "Identity" input: "clip_by_global_norm/mul_19" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_20" op: "Mul" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_19" op: "Identity" input: "clip_by_global_norm/mul_20" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_21" op: "Mul" input: "gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_20" op: "Identity" input: "clip_by_global_norm/mul_21" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_22" op: "Mul" input: "gradients/bert/encoder/layer_1/attention/self/query/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_21" op: "Identity" input: "clip_by_global_norm/mul_22" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_23" op: "Mul" input: "gradients/bert/encoder/layer_1/attention/self/query/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_22" op: "Identity" input: "clip_by_global_norm/mul_23" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_24" op: "Mul" input: "gradients/bert/encoder/layer_1/attention/self/key/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_23" op: "Identity" input: "clip_by_global_norm/mul_24" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_25" op: "Mul" input: "gradients/bert/encoder/layer_1/attention/self/key/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_24" op: "Identity" input: "clip_by_global_norm/mul_25" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_26" op: "Mul" input: "gradients/bert/encoder/layer_1/attention/self/value/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_25" op: "Identity" input: "clip_by_global_norm/mul_26" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_27" op: "Mul" input: "gradients/bert/encoder/layer_1/attention/self/value/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_26" op: "Identity" input: "clip_by_global_norm/mul_27" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_28" op: "Mul" input: "gradients/bert/encoder/layer_1/attention/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_27" op: "Identity" input: "clip_by_global_norm/mul_28" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_29" op: "Mul" input: "gradients/bert/encoder/layer_1/attention/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_28" op: "Identity" input: "clip_by_global_norm/mul_29" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_30" op: "Mul" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_29" op: "Identity" input: "clip_by_global_norm/mul_30" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_31" op: "Mul" input: "gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_30" op: "Identity" input: "clip_by_global_norm/mul_31" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_32" op: "Mul" input: "gradients/bert/encoder/layer_1/intermediate/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_31" op: "Identity" input: "clip_by_global_norm/mul_32" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_33" op: "Mul" input: "gradients/bert/encoder/layer_1/intermediate/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_32" op: "Identity" input: "clip_by_global_norm/mul_33" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_34" op: "Mul" input: "gradients/bert/encoder/layer_1/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_33" op: "Identity" input: "clip_by_global_norm/mul_34" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_35" op: "Mul" input: "gradients/bert/encoder/layer_1/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_34" op: "Identity" input: "clip_by_global_norm/mul_35" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_36" op: "Mul" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_35" op: "Identity" input: "clip_by_global_norm/mul_36" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_37" op: "Mul" input: "gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_36" op: "Identity" input: "clip_by_global_norm/mul_37" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_38" op: "Mul" input: "gradients/bert/encoder/layer_2/attention/self/query/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_37" op: "Identity" input: "clip_by_global_norm/mul_38" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_39" op: "Mul" input: "gradients/bert/encoder/layer_2/attention/self/query/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_38" op: "Identity" input: "clip_by_global_norm/mul_39" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_40" op: "Mul" input: "gradients/bert/encoder/layer_2/attention/self/key/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_39" op: "Identity" input: "clip_by_global_norm/mul_40" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_41" op: "Mul" input: "gradients/bert/encoder/layer_2/attention/self/key/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_40" op: "Identity" input: "clip_by_global_norm/mul_41" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_42" op: "Mul" input: "gradients/bert/encoder/layer_2/attention/self/value/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_41" op: "Identity" input: "clip_by_global_norm/mul_42" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_43" op: "Mul" input: "gradients/bert/encoder/layer_2/attention/self/value/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_42" op: "Identity" input: "clip_by_global_norm/mul_43" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_44" op: "Mul" input: "gradients/bert/encoder/layer_2/attention/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_43" op: "Identity" input: "clip_by_global_norm/mul_44" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_45" op: "Mul" input: "gradients/bert/encoder/layer_2/attention/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_44" op: "Identity" input: "clip_by_global_norm/mul_45" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_46" op: "Mul" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_45" op: "Identity" input: "clip_by_global_norm/mul_46" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_47" op: "Mul" input: "gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_46" op: "Identity" input: "clip_by_global_norm/mul_47" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_48" op: "Mul" input: "gradients/bert/encoder/layer_2/intermediate/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_47" op: "Identity" input: "clip_by_global_norm/mul_48" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_49" op: "Mul" input: "gradients/bert/encoder/layer_2/intermediate/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_48" op: "Identity" input: "clip_by_global_norm/mul_49" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_50" op: "Mul" input: "gradients/bert/encoder/layer_2/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_49" op: "Identity" input: "clip_by_global_norm/mul_50" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_51" op: "Mul" input: "gradients/bert/encoder/layer_2/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_50" op: "Identity" input: "clip_by_global_norm/mul_51" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_52" op: "Mul" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_51" op: "Identity" input: "clip_by_global_norm/mul_52" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_53" op: "Mul" input: "gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_52" op: "Identity" input: "clip_by_global_norm/mul_53" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_54" op: "Mul" input: "gradients/bert/encoder/layer_3/attention/self/query/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_53" op: "Identity" input: "clip_by_global_norm/mul_54" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_55" op: "Mul" input: "gradients/bert/encoder/layer_3/attention/self/query/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_54" op: "Identity" input: "clip_by_global_norm/mul_55" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_56" op: "Mul" input: "gradients/bert/encoder/layer_3/attention/self/key/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_55" op: "Identity" input: "clip_by_global_norm/mul_56" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_57" op: "Mul" input: "gradients/bert/encoder/layer_3/attention/self/key/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_56" op: "Identity" input: "clip_by_global_norm/mul_57" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_58" op: "Mul" input: "gradients/bert/encoder/layer_3/attention/self/value/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_57" op: "Identity" input: "clip_by_global_norm/mul_58" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_59" op: "Mul" input: "gradients/bert/encoder/layer_3/attention/self/value/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_58" op: "Identity" input: "clip_by_global_norm/mul_59" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_60" op: "Mul" input: "gradients/bert/encoder/layer_3/attention/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_59" op: "Identity" input: "clip_by_global_norm/mul_60" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_61" op: "Mul" input: "gradients/bert/encoder/layer_3/attention/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_60" op: "Identity" input: "clip_by_global_norm/mul_61" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_62" op: "Mul" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_61" op: "Identity" input: "clip_by_global_norm/mul_62" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_63" op: "Mul" input: "gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_62" op: "Identity" input: "clip_by_global_norm/mul_63" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_64" op: "Mul" input: "gradients/bert/encoder/layer_3/intermediate/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_63" op: "Identity" input: "clip_by_global_norm/mul_64" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_65" op: "Mul" input: "gradients/bert/encoder/layer_3/intermediate/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_64" op: "Identity" input: "clip_by_global_norm/mul_65" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_66" op: "Mul" input: "gradients/bert/encoder/layer_3/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_65" op: "Identity" input: "clip_by_global_norm/mul_66" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_67" op: "Mul" input: "gradients/bert/encoder/layer_3/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_66" op: "Identity" input: "clip_by_global_norm/mul_67" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_68" op: "Mul" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_67" op: "Identity" input: "clip_by_global_norm/mul_68" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_69" op: "Mul" input: "gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_68" op: "Identity" input: "clip_by_global_norm/mul_69" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_70" op: "Mul" input: "gradients/bert/encoder/layer_4/attention/self/query/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_69" op: "Identity" input: "clip_by_global_norm/mul_70" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_71" op: "Mul" input: "gradients/bert/encoder/layer_4/attention/self/query/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_70" op: "Identity" input: "clip_by_global_norm/mul_71" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_72" op: "Mul" input: "gradients/bert/encoder/layer_4/attention/self/key/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_71" op: "Identity" input: "clip_by_global_norm/mul_72" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_73" op: "Mul" input: "gradients/bert/encoder/layer_4/attention/self/key/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_72" op: "Identity" input: "clip_by_global_norm/mul_73" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_74" op: "Mul" input: "gradients/bert/encoder/layer_4/attention/self/value/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_73" op: "Identity" input: "clip_by_global_norm/mul_74" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_75" op: "Mul" input: "gradients/bert/encoder/layer_4/attention/self/value/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_74" op: "Identity" input: "clip_by_global_norm/mul_75" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_76" op: "Mul" input: "gradients/bert/encoder/layer_4/attention/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_75" op: "Identity" input: "clip_by_global_norm/mul_76" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_77" op: "Mul" input: "gradients/bert/encoder/layer_4/attention/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_76" op: "Identity" input: "clip_by_global_norm/mul_77" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_78" op: "Mul" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_77" op: "Identity" input: "clip_by_global_norm/mul_78" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_79" op: "Mul" input: "gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_78" op: "Identity" input: "clip_by_global_norm/mul_79" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_80" op: "Mul" input: "gradients/bert/encoder/layer_4/intermediate/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_79" op: "Identity" input: "clip_by_global_norm/mul_80" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_81" op: "Mul" input: "gradients/bert/encoder/layer_4/intermediate/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_80" op: "Identity" input: "clip_by_global_norm/mul_81" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_82" op: "Mul" input: "gradients/bert/encoder/layer_4/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_81" op: "Identity" input: "clip_by_global_norm/mul_82" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_83" op: "Mul" input: "gradients/bert/encoder/layer_4/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_82" op: "Identity" input: "clip_by_global_norm/mul_83" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_84" op: "Mul" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_83" op: "Identity" input: "clip_by_global_norm/mul_84" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_85" op: "Mul" input: "gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_84" op: "Identity" input: "clip_by_global_norm/mul_85" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_86" op: "Mul" input: "gradients/bert/encoder/layer_5/attention/self/query/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_85" op: "Identity" input: "clip_by_global_norm/mul_86" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_87" op: "Mul" input: "gradients/bert/encoder/layer_5/attention/self/query/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_86" op: "Identity" input: "clip_by_global_norm/mul_87" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_88" op: "Mul" input: "gradients/bert/encoder/layer_5/attention/self/key/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_87" op: "Identity" input: "clip_by_global_norm/mul_88" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_89" op: "Mul" input: "gradients/bert/encoder/layer_5/attention/self/key/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_88" op: "Identity" input: "clip_by_global_norm/mul_89" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_90" op: "Mul" input: "gradients/bert/encoder/layer_5/attention/self/value/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_89" op: "Identity" input: "clip_by_global_norm/mul_90" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_91" op: "Mul" input: "gradients/bert/encoder/layer_5/attention/self/value/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_90" op: "Identity" input: "clip_by_global_norm/mul_91" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_92" op: "Mul" input: "gradients/bert/encoder/layer_5/attention/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_91" op: "Identity" input: "clip_by_global_norm/mul_92" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_93" op: "Mul" input: "gradients/bert/encoder/layer_5/attention/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_92" op: "Identity" input: "clip_by_global_norm/mul_93" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_94" op: "Mul" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_93" op: "Identity" input: "clip_by_global_norm/mul_94" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_95" op: "Mul" input: "gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_94" op: "Identity" input: "clip_by_global_norm/mul_95" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_96" op: "Mul" input: "gradients/bert/encoder/layer_5/intermediate/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_95" op: "Identity" input: "clip_by_global_norm/mul_96" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_97" op: "Mul" input: "gradients/bert/encoder/layer_5/intermediate/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_96" op: "Identity" input: "clip_by_global_norm/mul_97" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_98" op: "Mul" input: "gradients/bert/encoder/layer_5/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_97" op: "Identity" input: "clip_by_global_norm/mul_98" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_99" op: "Mul" input: "gradients/bert/encoder/layer_5/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_98" op: "Identity" input: "clip_by_global_norm/mul_99" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_100" op: "Mul" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_99" op: "Identity" input: "clip_by_global_norm/mul_100" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_101" op: "Mul" input: "gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_100" op: "Identity" input: "clip_by_global_norm/mul_101" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_102" op: "Mul" input: "gradients/bert/encoder/layer_6/attention/self/query/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_101" op: "Identity" input: "clip_by_global_norm/mul_102" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_103" op: "Mul" input: "gradients/bert/encoder/layer_6/attention/self/query/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_102" op: "Identity" input: "clip_by_global_norm/mul_103" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_104" op: "Mul" input: "gradients/bert/encoder/layer_6/attention/self/key/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_103" op: "Identity" input: "clip_by_global_norm/mul_104" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_105" op: "Mul" input: "gradients/bert/encoder/layer_6/attention/self/key/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_104" op: "Identity" input: "clip_by_global_norm/mul_105" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_106" op: "Mul" input: "gradients/bert/encoder/layer_6/attention/self/value/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_105" op: "Identity" input: "clip_by_global_norm/mul_106" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_107" op: "Mul" input: "gradients/bert/encoder/layer_6/attention/self/value/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_106" op: "Identity" input: "clip_by_global_norm/mul_107" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_108" op: "Mul" input: "gradients/bert/encoder/layer_6/attention/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_107" op: "Identity" input: "clip_by_global_norm/mul_108" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_109" op: "Mul" input: "gradients/bert/encoder/layer_6/attention/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_108" op: "Identity" input: "clip_by_global_norm/mul_109" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_110" op: "Mul" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_109" op: "Identity" input: "clip_by_global_norm/mul_110" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_111" op: "Mul" input: "gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_110" op: "Identity" input: "clip_by_global_norm/mul_111" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_112" op: "Mul" input: "gradients/bert/encoder/layer_6/intermediate/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_111" op: "Identity" input: "clip_by_global_norm/mul_112" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_113" op: "Mul" input: "gradients/bert/encoder/layer_6/intermediate/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_112" op: "Identity" input: "clip_by_global_norm/mul_113" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_114" op: "Mul" input: "gradients/bert/encoder/layer_6/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_113" op: "Identity" input: "clip_by_global_norm/mul_114" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_115" op: "Mul" input: "gradients/bert/encoder/layer_6/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_114" op: "Identity" input: "clip_by_global_norm/mul_115" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_116" op: "Mul" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_115" op: "Identity" input: "clip_by_global_norm/mul_116" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_117" op: "Mul" input: "gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_116" op: "Identity" input: "clip_by_global_norm/mul_117" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_118" op: "Mul" input: "gradients/bert/encoder/layer_7/attention/self/query/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_117" op: "Identity" input: "clip_by_global_norm/mul_118" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_119" op: "Mul" input: "gradients/bert/encoder/layer_7/attention/self/query/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_118" op: "Identity" input: "clip_by_global_norm/mul_119" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_120" op: "Mul" input: "gradients/bert/encoder/layer_7/attention/self/key/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_119" op: "Identity" input: "clip_by_global_norm/mul_120" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_121" op: "Mul" input: "gradients/bert/encoder/layer_7/attention/self/key/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_120" op: "Identity" input: "clip_by_global_norm/mul_121" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_122" op: "Mul" input: "gradients/bert/encoder/layer_7/attention/self/value/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_121" op: "Identity" input: "clip_by_global_norm/mul_122" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_123" op: "Mul" input: "gradients/bert/encoder/layer_7/attention/self/value/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_122" op: "Identity" input: "clip_by_global_norm/mul_123" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_124" op: "Mul" input: "gradients/bert/encoder/layer_7/attention/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_123" op: "Identity" input: "clip_by_global_norm/mul_124" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_125" op: "Mul" input: "gradients/bert/encoder/layer_7/attention/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_124" op: "Identity" input: "clip_by_global_norm/mul_125" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_126" op: "Mul" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_125" op: "Identity" input: "clip_by_global_norm/mul_126" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_127" op: "Mul" input: "gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_126" op: "Identity" input: "clip_by_global_norm/mul_127" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_128" op: "Mul" input: "gradients/bert/encoder/layer_7/intermediate/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_127" op: "Identity" input: "clip_by_global_norm/mul_128" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_129" op: "Mul" input: "gradients/bert/encoder/layer_7/intermediate/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_128" op: "Identity" input: "clip_by_global_norm/mul_129" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_130" op: "Mul" input: "gradients/bert/encoder/layer_7/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_129" op: "Identity" input: "clip_by_global_norm/mul_130" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_131" op: "Mul" input: "gradients/bert/encoder/layer_7/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_130" op: "Identity" input: "clip_by_global_norm/mul_131" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_132" op: "Mul" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_131" op: "Identity" input: "clip_by_global_norm/mul_132" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_133" op: "Mul" input: "gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_132" op: "Identity" input: "clip_by_global_norm/mul_133" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_134" op: "Mul" input: "gradients/bert/encoder/layer_8/attention/self/query/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_133" op: "Identity" input: "clip_by_global_norm/mul_134" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_135" op: "Mul" input: "gradients/bert/encoder/layer_8/attention/self/query/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_134" op: "Identity" input: "clip_by_global_norm/mul_135" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_136" op: "Mul" input: "gradients/bert/encoder/layer_8/attention/self/key/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_135" op: "Identity" input: "clip_by_global_norm/mul_136" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_137" op: "Mul" input: "gradients/bert/encoder/layer_8/attention/self/key/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_136" op: "Identity" input: "clip_by_global_norm/mul_137" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_138" op: "Mul" input: "gradients/bert/encoder/layer_8/attention/self/value/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_137" op: "Identity" input: "clip_by_global_norm/mul_138" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_139" op: "Mul" input: "gradients/bert/encoder/layer_8/attention/self/value/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_138" op: "Identity" input: "clip_by_global_norm/mul_139" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_140" op: "Mul" input: "gradients/bert/encoder/layer_8/attention/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_139" op: "Identity" input: "clip_by_global_norm/mul_140" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_141" op: "Mul" input: "gradients/bert/encoder/layer_8/attention/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_140" op: "Identity" input: "clip_by_global_norm/mul_141" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_142" op: "Mul" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_141" op: "Identity" input: "clip_by_global_norm/mul_142" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_143" op: "Mul" input: "gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_142" op: "Identity" input: "clip_by_global_norm/mul_143" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_144" op: "Mul" input: "gradients/bert/encoder/layer_8/intermediate/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_143" op: "Identity" input: "clip_by_global_norm/mul_144" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_145" op: "Mul" input: "gradients/bert/encoder/layer_8/intermediate/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_144" op: "Identity" input: "clip_by_global_norm/mul_145" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_146" op: "Mul" input: "gradients/bert/encoder/layer_8/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_145" op: "Identity" input: "clip_by_global_norm/mul_146" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_147" op: "Mul" input: "gradients/bert/encoder/layer_8/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_146" op: "Identity" input: "clip_by_global_norm/mul_147" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_148" op: "Mul" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_147" op: "Identity" input: "clip_by_global_norm/mul_148" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_149" op: "Mul" input: "gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_148" op: "Identity" input: "clip_by_global_norm/mul_149" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_150" op: "Mul" input: "gradients/bert/encoder/layer_9/attention/self/query/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_149" op: "Identity" input: "clip_by_global_norm/mul_150" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_151" op: "Mul" input: "gradients/bert/encoder/layer_9/attention/self/query/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_150" op: "Identity" input: "clip_by_global_norm/mul_151" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_152" op: "Mul" input: "gradients/bert/encoder/layer_9/attention/self/key/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_151" op: "Identity" input: "clip_by_global_norm/mul_152" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_153" op: "Mul" input: "gradients/bert/encoder/layer_9/attention/self/key/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_152" op: "Identity" input: "clip_by_global_norm/mul_153" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_154" op: "Mul" input: "gradients/bert/encoder/layer_9/attention/self/value/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_153" op: "Identity" input: "clip_by_global_norm/mul_154" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_155" op: "Mul" input: "gradients/bert/encoder/layer_9/attention/self/value/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_154" op: "Identity" input: "clip_by_global_norm/mul_155" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_156" op: "Mul" input: "gradients/bert/encoder/layer_9/attention/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_155" op: "Identity" input: "clip_by_global_norm/mul_156" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_157" op: "Mul" input: "gradients/bert/encoder/layer_9/attention/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_156" op: "Identity" input: "clip_by_global_norm/mul_157" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_158" op: "Mul" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_157" op: "Identity" input: "clip_by_global_norm/mul_158" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_159" op: "Mul" input: "gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_158" op: "Identity" input: "clip_by_global_norm/mul_159" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_160" op: "Mul" input: "gradients/bert/encoder/layer_9/intermediate/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_159" op: "Identity" input: "clip_by_global_norm/mul_160" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_161" op: "Mul" input: "gradients/bert/encoder/layer_9/intermediate/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_160" op: "Identity" input: "clip_by_global_norm/mul_161" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_162" op: "Mul" input: "gradients/bert/encoder/layer_9/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_161" op: "Identity" input: "clip_by_global_norm/mul_162" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_163" op: "Mul" input: "gradients/bert/encoder/layer_9/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_162" op: "Identity" input: "clip_by_global_norm/mul_163" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_164" op: "Mul" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_163" op: "Identity" input: "clip_by_global_norm/mul_164" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_165" op: "Mul" input: "gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_164" op: "Identity" input: "clip_by_global_norm/mul_165" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_166" op: "Mul" input: "gradients/bert/encoder/layer_10/attention/self/query/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_165" op: "Identity" input: "clip_by_global_norm/mul_166" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_167" op: "Mul" input: "gradients/bert/encoder/layer_10/attention/self/query/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_166" op: "Identity" input: "clip_by_global_norm/mul_167" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_168" op: "Mul" input: "gradients/bert/encoder/layer_10/attention/self/key/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_167" op: "Identity" input: "clip_by_global_norm/mul_168" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_169" op: "Mul" input: "gradients/bert/encoder/layer_10/attention/self/key/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_168" op: "Identity" input: "clip_by_global_norm/mul_169" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_170" op: "Mul" input: "gradients/bert/encoder/layer_10/attention/self/value/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_169" op: "Identity" input: "clip_by_global_norm/mul_170" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_171" op: "Mul" input: "gradients/bert/encoder/layer_10/attention/self/value/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_170" op: "Identity" input: "clip_by_global_norm/mul_171" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_172" op: "Mul" input: "gradients/bert/encoder/layer_10/attention/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_171" op: "Identity" input: "clip_by_global_norm/mul_172" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_173" op: "Mul" input: "gradients/bert/encoder/layer_10/attention/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_172" op: "Identity" input: "clip_by_global_norm/mul_173" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_174" op: "Mul" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_173" op: "Identity" input: "clip_by_global_norm/mul_174" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_175" op: "Mul" input: "gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_174" op: "Identity" input: "clip_by_global_norm/mul_175" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_176" op: "Mul" input: "gradients/bert/encoder/layer_10/intermediate/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_175" op: "Identity" input: "clip_by_global_norm/mul_176" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_177" op: "Mul" input: "gradients/bert/encoder/layer_10/intermediate/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_176" op: "Identity" input: "clip_by_global_norm/mul_177" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_178" op: "Mul" input: "gradients/bert/encoder/layer_10/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_177" op: "Identity" input: "clip_by_global_norm/mul_178" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_179" op: "Mul" input: "gradients/bert/encoder/layer_10/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_178" op: "Identity" input: "clip_by_global_norm/mul_179" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_180" op: "Mul" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_179" op: "Identity" input: "clip_by_global_norm/mul_180" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_181" op: "Mul" input: "gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_180" op: "Identity" input: "clip_by_global_norm/mul_181" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_182" op: "Mul" input: "gradients/bert/encoder/layer_11/attention/self/query/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_181" op: "Identity" input: "clip_by_global_norm/mul_182" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_183" op: "Mul" input: "gradients/bert/encoder/layer_11/attention/self/query/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_182" op: "Identity" input: "clip_by_global_norm/mul_183" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_184" op: "Mul" input: "gradients/bert/encoder/layer_11/attention/self/key/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_183" op: "Identity" input: "clip_by_global_norm/mul_184" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_185" op: "Mul" input: "gradients/bert/encoder/layer_11/attention/self/key/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_184" op: "Identity" input: "clip_by_global_norm/mul_185" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_186" op: "Mul" input: "gradients/bert/encoder/layer_11/attention/self/value/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_185" op: "Identity" input: "clip_by_global_norm/mul_186" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_187" op: "Mul" input: "gradients/bert/encoder/layer_11/attention/self/value/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_186" op: "Identity" input: "clip_by_global_norm/mul_187" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_188" op: "Mul" input: "gradients/bert/encoder/layer_11/attention/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_187" op: "Identity" input: "clip_by_global_norm/mul_188" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_189" op: "Mul" input: "gradients/bert/encoder/layer_11/attention/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_188" op: "Identity" input: "clip_by_global_norm/mul_189" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_190" op: "Mul" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_189" op: "Identity" input: "clip_by_global_norm/mul_190" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_191" op: "Mul" input: "gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_190" op: "Identity" input: "clip_by_global_norm/mul_191" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_192" op: "Mul" input: "gradients/bert/encoder/layer_11/intermediate/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_191" op: "Identity" input: "clip_by_global_norm/mul_192" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_193" op: "Mul" input: "gradients/bert/encoder/layer_11/intermediate/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_192" op: "Identity" input: "clip_by_global_norm/mul_193" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_194" op: "Mul" input: "gradients/bert/encoder/layer_11/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_193" op: "Identity" input: "clip_by_global_norm/mul_194" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_195" op: "Mul" input: "gradients/bert/encoder/layer_11/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_194" op: "Identity" input: "clip_by_global_norm/mul_195" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_196" op: "Mul" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_195" op: "Identity" input: "clip_by_global_norm/mul_196" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_197" op: "Mul" input: "gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_196" op: "Identity" input: "clip_by_global_norm/mul_197" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_198" op: "Mul" input: "gradients/bert/pooler/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/pooler/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_197" op: "Identity" input: "clip_by_global_norm/mul_198" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/pooler/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_199" op: "Mul" input: "gradients/bert/pooler/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/pooler/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_198" op: "Identity" input: "clip_by_global_norm/mul_199" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/pooler/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_200" op: "Mul" input: "gradients/cls/predictions/transform/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/transform/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_199" op: "Identity" input: "clip_by_global_norm/mul_200" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/transform/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_201" op: "Mul" input: "gradients/cls/predictions/transform/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/transform/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_200" op: "Identity" input: "clip_by_global_norm/mul_201" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/transform/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_202" op: "Mul" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/transform/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_201" op: "Identity" input: "clip_by_global_norm/mul_202" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/transform/LayerNorm/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_203" op: "Mul" input: "gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_202" op: "Identity" input: "clip_by_global_norm/mul_203" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/transform/LayerNorm/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_204" op: "Mul" input: "gradients/cls/predictions/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_203" op: "Identity" input: "clip_by_global_norm/mul_204" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/cls/predictions/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } } } } } } node { name: "clip_by_global_norm/mul_205" op: "Mul" input: "gradients/cls/seq_relationship/MatMul_grad/MatMul_1" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/cls/seq_relationship/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_204" op: "Identity" input: "clip_by_global_norm/mul_205" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/cls/seq_relationship/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_206" op: "Mul" input: "gradients/cls/seq_relationship/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/cls/seq_relationship/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_205" op: "Identity" input: "clip_by_global_norm/mul_206" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/cls/seq_relationship/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "bert/embeddings/word_embeddings/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "Dq\000\000\000\003\000\000" } } } } node { name: "bert/embeddings/word_embeddings/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/embeddings/word_embeddings/adam_m/Initializer/zeros" op: "Fill" input: "bert/embeddings/word_embeddings/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/embeddings/word_embeddings/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/embeddings/word_embeddings/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 28996 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/embeddings/word_embeddings/adam_m/Assign" op: "Assign" input: "bert/embeddings/word_embeddings/adam_m" input: "bert/embeddings/word_embeddings/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/embeddings/word_embeddings/adam_m/read" op: "Identity" input: "bert/embeddings/word_embeddings/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } dim { size: 768 } } } } } } node { name: "bert/embeddings/word_embeddings/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "Dq\000\000\000\003\000\000" } } } } node { name: "bert/embeddings/word_embeddings/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/embeddings/word_embeddings/adam_v/Initializer/zeros" op: "Fill" input: "bert/embeddings/word_embeddings/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/embeddings/word_embeddings/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/embeddings/word_embeddings/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 28996 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/embeddings/word_embeddings/adam_v/Assign" op: "Assign" input: "bert/embeddings/word_embeddings/adam_v" input: "bert/embeddings/word_embeddings/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/embeddings/word_embeddings/adam_v/read" op: "Identity" input: "bert/embeddings/word_embeddings/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } dim { size: 768 } } } } } } node { name: "Mul_4/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_4" op: "Mul" input: "Mul_4/x" input: "bert/embeddings/word_embeddings/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } dim { size: 768 } } } } } } node { name: "Mul_5/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_5/strided_slice/stack" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "Mul_5/strided_slice/stack_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "Mul_5/strided_slice/stack_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "Mul_5/strided_slice" op: "StridedSlice" input: "gradients/Shape_1" input: "Mul_5/strided_slice/stack" input: "Mul_5/strided_slice/stack_1" input: "Mul_5/strided_slice/stack_2" attr { key: "Index" value { type: DT_INT32 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "begin_mask" value { i: 0 } } attr { key: "ellipsis_mask" value { i: 0 } } attr { key: "end_mask" value { i: 0 } } attr { key: "new_axis_mask" value { i: 0 } } attr { key: "shrink_axis_mask" value { i: 1 } } } node { name: "Mul_5/y" op: "UnsortedSegmentSum" input: "clip_by_global_norm/clip_by_global_norm/_0" input: "gradients/concat_1" input: "Mul_5/strided_slice" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tindices" value { type: DT_INT32 } } attr { key: "Tnumsegments" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } dim { size: 768 } } } } } } node { name: "Mul_5" op: "Mul" input: "Mul_5/x" input: "Mul_5/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } dim { size: 768 } } } } } } node { name: "add_3" op: "Add" input: "Mul_4" input: "Mul_5" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } dim { size: 768 } } } } } } node { name: "Mul_6/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_6" op: "Mul" input: "Mul_6/x" input: "bert/embeddings/word_embeddings/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } dim { size: 768 } } } } } } node { name: "Square/strided_slice/stack" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "Square/strided_slice/stack_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "Square/strided_slice/stack_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "Square/strided_slice" op: "StridedSlice" input: "gradients/Shape_1" input: "Square/strided_slice/stack" input: "Square/strided_slice/stack_1" input: "Square/strided_slice/stack_2" attr { key: "Index" value { type: DT_INT32 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "begin_mask" value { i: 0 } } attr { key: "ellipsis_mask" value { i: 0 } } attr { key: "end_mask" value { i: 0 } } attr { key: "new_axis_mask" value { i: 0 } } attr { key: "shrink_axis_mask" value { i: 1 } } } node { name: "Square/x" op: "UnsortedSegmentSum" input: "clip_by_global_norm/clip_by_global_norm/_0" input: "gradients/concat_1" input: "Square/strided_slice" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tindices" value { type: DT_INT32 } } attr { key: "Tnumsegments" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } dim { size: 768 } } } } } } node { name: "Square" op: "Square" input: "Square/x" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } dim { size: 768 } } } } } } node { name: "Mul_7/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_7" op: "Mul" input: "Mul_7/x" input: "Square" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } dim { size: 768 } } } } } } node { name: "add_4" op: "Add" input: "Mul_6" input: "Mul_7" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } dim { size: 768 } } } } } } node { name: "Sqrt" op: "Sqrt" input: "add_4" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } dim { size: 768 } } } } } } node { name: "add_5/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_5" op: "Add" input: "Sqrt" input: "add_5/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } dim { size: 768 } } } } } } node { name: "truediv_1" op: "RealDiv" input: "add_3" input: "add_5" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } dim { size: 768 } } } } } } node { name: "mul_8/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_8" op: "Mul" input: "mul_8/x" input: "bert/embeddings/word_embeddings/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } dim { size: 768 } } } } } } node { name: "add_6" op: "Add" input: "truediv_1" input: "mul_8" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } dim { size: 768 } } } } } } node { name: "mul_9" op: "Mul" input: "add_2" input: "add_6" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } dim { size: 768 } } } } } } node { name: "sub_1" op: "Sub" input: "bert/embeddings/word_embeddings/read" input: "mul_9" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } dim { size: 768 } } } } } } node { name: "Assign_206" op: "Assign" input: "bert/embeddings/word_embeddings" input: "sub_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_207" op: "Assign" input: "bert/embeddings/word_embeddings/adam_m" input: "add_3" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_208" op: "Assign" input: "bert/embeddings/word_embeddings/adam_v" input: "add_4" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/embeddings/token_type_embeddings/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\002\000\000\000\000\003\000\000" } } } } node { name: "bert/embeddings/token_type_embeddings/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/embeddings/token_type_embeddings/adam_m/Initializer/zeros" op: "Fill" input: "bert/embeddings/token_type_embeddings/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/embeddings/token_type_embeddings/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/embeddings/token_type_embeddings/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 2 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/embeddings/token_type_embeddings/adam_m/Assign" op: "Assign" input: "bert/embeddings/token_type_embeddings/adam_m" input: "bert/embeddings/token_type_embeddings/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/embeddings/token_type_embeddings/adam_m/read" op: "Identity" input: "bert/embeddings/token_type_embeddings/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "bert/embeddings/token_type_embeddings/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\002\000\000\000\000\003\000\000" } } } } node { name: "bert/embeddings/token_type_embeddings/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/embeddings/token_type_embeddings/adam_v/Initializer/zeros" op: "Fill" input: "bert/embeddings/token_type_embeddings/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/embeddings/token_type_embeddings/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/embeddings/token_type_embeddings/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 2 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/embeddings/token_type_embeddings/adam_v/Assign" op: "Assign" input: "bert/embeddings/token_type_embeddings/adam_v" input: "bert/embeddings/token_type_embeddings/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/embeddings/token_type_embeddings/adam_v/read" op: "Identity" input: "bert/embeddings/token_type_embeddings/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "Mul_10/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_10" op: "Mul" input: "Mul_10/x" input: "bert/embeddings/token_type_embeddings/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "Mul_11/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_11" op: "Mul" input: "Mul_11/x" input: "clip_by_global_norm/clip_by_global_norm/_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "add_7" op: "Add" input: "Mul_10" input: "Mul_11" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "Mul_12/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_12" op: "Mul" input: "Mul_12/x" input: "bert/embeddings/token_type_embeddings/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "Square_1" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "Mul_13/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_13" op: "Mul" input: "Mul_13/x" input: "Square_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "add_8" op: "Add" input: "Mul_12" input: "Mul_13" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "Sqrt_1" op: "Sqrt" input: "add_8" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "add_9/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_9" op: "Add" input: "Sqrt_1" input: "add_9/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "truediv_2" op: "RealDiv" input: "add_7" input: "add_9" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "mul_14/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_14" op: "Mul" input: "mul_14/x" input: "bert/embeddings/token_type_embeddings/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "add_10" op: "Add" input: "truediv_2" input: "mul_14" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "mul_15" op: "Mul" input: "add_2" input: "add_10" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "sub_2" op: "Sub" input: "bert/embeddings/token_type_embeddings/read" input: "mul_15" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "Assign_209" op: "Assign" input: "bert/embeddings/token_type_embeddings" input: "sub_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_210" op: "Assign" input: "bert/embeddings/token_type_embeddings/adam_m" input: "add_7" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_211" op: "Assign" input: "bert/embeddings/token_type_embeddings/adam_v" input: "add_8" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/embeddings/position_embeddings/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\002\000\000\000\003\000\000" } } } } node { name: "bert/embeddings/position_embeddings/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/embeddings/position_embeddings/adam_m/Initializer/zeros" op: "Fill" input: "bert/embeddings/position_embeddings/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/embeddings/position_embeddings/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/embeddings/position_embeddings/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 512 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/embeddings/position_embeddings/adam_m/Assign" op: "Assign" input: "bert/embeddings/position_embeddings/adam_m" input: "bert/embeddings/position_embeddings/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/embeddings/position_embeddings/adam_m/read" op: "Identity" input: "bert/embeddings/position_embeddings/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } } node { name: "bert/embeddings/position_embeddings/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\002\000\000\000\003\000\000" } } } } node { name: "bert/embeddings/position_embeddings/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/embeddings/position_embeddings/adam_v/Initializer/zeros" op: "Fill" input: "bert/embeddings/position_embeddings/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/embeddings/position_embeddings/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/embeddings/position_embeddings/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 512 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/embeddings/position_embeddings/adam_v/Assign" op: "Assign" input: "bert/embeddings/position_embeddings/adam_v" input: "bert/embeddings/position_embeddings/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/embeddings/position_embeddings/adam_v/read" op: "Identity" input: "bert/embeddings/position_embeddings/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } } node { name: "Mul_16/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_16" op: "Mul" input: "Mul_16/x" input: "bert/embeddings/position_embeddings/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } } node { name: "Mul_17/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_17" op: "Mul" input: "Mul_17/x" input: "clip_by_global_norm/clip_by_global_norm/_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } } node { name: "add_11" op: "Add" input: "Mul_16" input: "Mul_17" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } } node { name: "Mul_18/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_18" op: "Mul" input: "Mul_18/x" input: "bert/embeddings/position_embeddings/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } } node { name: "Square_2" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } } node { name: "Mul_19/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_19" op: "Mul" input: "Mul_19/x" input: "Square_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } } node { name: "add_12" op: "Add" input: "Mul_18" input: "Mul_19" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } } node { name: "Sqrt_2" op: "Sqrt" input: "add_12" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } } node { name: "add_13/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_13" op: "Add" input: "Sqrt_2" input: "add_13/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } } node { name: "truediv_3" op: "RealDiv" input: "add_11" input: "add_13" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } } node { name: "mul_20/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_20" op: "Mul" input: "mul_20/x" input: "bert/embeddings/position_embeddings/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } } node { name: "add_14" op: "Add" input: "truediv_3" input: "mul_20" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } } node { name: "mul_21" op: "Mul" input: "add_2" input: "add_14" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } } node { name: "sub_3" op: "Sub" input: "bert/embeddings/position_embeddings/read" input: "mul_21" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } } node { name: "Assign_212" op: "Assign" input: "bert/embeddings/position_embeddings" input: "sub_3" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_213" op: "Assign" input: "bert/embeddings/position_embeddings/adam_m" input: "add_11" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_214" op: "Assign" input: "bert/embeddings/position_embeddings/adam_v" input: "add_12" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/embeddings/LayerNorm/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/embeddings/LayerNorm/beta/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/embeddings/LayerNorm/beta/adam_m/Assign" op: "Assign" input: "bert/embeddings/LayerNorm/beta/adam_m" input: "bert/embeddings/LayerNorm/beta/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/embeddings/LayerNorm/beta/adam_m/read" op: "Identity" input: "bert/embeddings/LayerNorm/beta/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/embeddings/LayerNorm/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/embeddings/LayerNorm/beta/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/embeddings/LayerNorm/beta/adam_v/Assign" op: "Assign" input: "bert/embeddings/LayerNorm/beta/adam_v" input: "bert/embeddings/LayerNorm/beta/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/embeddings/LayerNorm/beta/adam_v/read" op: "Identity" input: "bert/embeddings/LayerNorm/beta/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_22/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_22" op: "Mul" input: "Mul_22/x" input: "bert/embeddings/LayerNorm/beta/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_23/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_23" op: "Mul" input: "Mul_23/x" input: "clip_by_global_norm/clip_by_global_norm/_3" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_15" op: "Add" input: "Mul_22" input: "Mul_23" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_24/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_24" op: "Mul" input: "Mul_24/x" input: "bert/embeddings/LayerNorm/beta/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_3" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_3" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_25/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_25" op: "Mul" input: "Mul_25/x" input: "Square_3" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_16" op: "Add" input: "Mul_24" input: "Mul_25" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_3" op: "Sqrt" input: "add_16" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_17/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_17" op: "Add" input: "Sqrt_3" input: "add_17/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_4" op: "RealDiv" input: "add_15" input: "add_17" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_26" op: "Mul" input: "add_2" input: "truediv_4" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_4" op: "Sub" input: "bert/embeddings/LayerNorm/beta/read" input: "mul_26" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_215" op: "Assign" input: "bert/embeddings/LayerNorm/beta" input: "sub_4" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_216" op: "Assign" input: "bert/embeddings/LayerNorm/beta/adam_m" input: "add_15" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_217" op: "Assign" input: "bert/embeddings/LayerNorm/beta/adam_v" input: "add_16" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/embeddings/LayerNorm/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/embeddings/LayerNorm/gamma/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/embeddings/LayerNorm/gamma/adam_m/Assign" op: "Assign" input: "bert/embeddings/LayerNorm/gamma/adam_m" input: "bert/embeddings/LayerNorm/gamma/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/embeddings/LayerNorm/gamma/adam_m/read" op: "Identity" input: "bert/embeddings/LayerNorm/gamma/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/embeddings/LayerNorm/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/embeddings/LayerNorm/gamma/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/embeddings/LayerNorm/gamma/adam_v/Assign" op: "Assign" input: "bert/embeddings/LayerNorm/gamma/adam_v" input: "bert/embeddings/LayerNorm/gamma/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/embeddings/LayerNorm/gamma/adam_v/read" op: "Identity" input: "bert/embeddings/LayerNorm/gamma/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_27/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_27" op: "Mul" input: "Mul_27/x" input: "bert/embeddings/LayerNorm/gamma/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_28/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_28" op: "Mul" input: "Mul_28/x" input: "clip_by_global_norm/clip_by_global_norm/_4" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_18" op: "Add" input: "Mul_27" input: "Mul_28" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_29/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_29" op: "Mul" input: "Mul_29/x" input: "bert/embeddings/LayerNorm/gamma/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_4" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_4" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_30/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_30" op: "Mul" input: "Mul_30/x" input: "Square_4" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_19" op: "Add" input: "Mul_29" input: "Mul_30" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_4" op: "Sqrt" input: "add_19" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_20/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_20" op: "Add" input: "Sqrt_4" input: "add_20/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_5" op: "RealDiv" input: "add_18" input: "add_20" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_31" op: "Mul" input: "add_2" input: "truediv_5" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_5" op: "Sub" input: "bert/embeddings/LayerNorm/gamma/read" input: "mul_31" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_218" op: "Assign" input: "bert/embeddings/LayerNorm/gamma" input: "sub_5" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_219" op: "Assign" input: "bert/embeddings/LayerNorm/gamma/adam_m" input: "add_18" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_220" op: "Assign" input: "bert/embeddings/LayerNorm/gamma/adam_v" input: "add_19" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_m" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_v" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_32/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_32" op: "Mul" input: "Mul_32/x" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_33/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_33" op: "Mul" input: "Mul_33/x" input: "clip_by_global_norm/clip_by_global_norm/_5" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_21" op: "Add" input: "Mul_32" input: "Mul_33" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_34/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_34" op: "Mul" input: "Mul_34/x" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_5" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_5" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_35/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_35" op: "Mul" input: "Mul_35/x" input: "Square_5" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_22" op: "Add" input: "Mul_34" input: "Mul_35" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_5" op: "Sqrt" input: "add_22" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_23/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_23" op: "Add" input: "Sqrt_5" input: "add_23/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_6" op: "RealDiv" input: "add_21" input: "add_23" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_36/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_36" op: "Mul" input: "mul_36/x" input: "bert/encoder/layer_0/attention/self/query/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_24" op: "Add" input: "truediv_6" input: "mul_36" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_37" op: "Mul" input: "add_2" input: "add_24" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_6" op: "Sub" input: "bert/encoder/layer_0/attention/self/query/kernel/read" input: "mul_37" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_221" op: "Assign" input: "bert/encoder/layer_0/attention/self/query/kernel" input: "sub_6" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_222" op: "Assign" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_m" input: "add_21" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_223" op: "Assign" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_v" input: "add_22" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/attention/self/query/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/self/query/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/attention/self/query/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_0/attention/self/query/bias/adam_m" input: "bert/encoder/layer_0/attention/self/query/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/attention/self/query/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_0/attention/self/query/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/self/query/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/self/query/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/attention/self/query/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_0/attention/self/query/bias/adam_v" input: "bert/encoder/layer_0/attention/self/query/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/attention/self/query/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_0/attention/self/query/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_38/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_38" op: "Mul" input: "Mul_38/x" input: "bert/encoder/layer_0/attention/self/query/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_39/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_39" op: "Mul" input: "Mul_39/x" input: "clip_by_global_norm/clip_by_global_norm/_6" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_25" op: "Add" input: "Mul_38" input: "Mul_39" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_40/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_40" op: "Mul" input: "Mul_40/x" input: "bert/encoder/layer_0/attention/self/query/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_6" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_6" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_41/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_41" op: "Mul" input: "Mul_41/x" input: "Square_6" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_26" op: "Add" input: "Mul_40" input: "Mul_41" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_6" op: "Sqrt" input: "add_26" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_27/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_27" op: "Add" input: "Sqrt_6" input: "add_27/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_7" op: "RealDiv" input: "add_25" input: "add_27" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_42" op: "Mul" input: "add_2" input: "truediv_7" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_7" op: "Sub" input: "bert/encoder/layer_0/attention/self/query/bias/read" input: "mul_42" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_224" op: "Assign" input: "bert/encoder/layer_0/attention/self/query/bias" input: "sub_7" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_225" op: "Assign" input: "bert/encoder/layer_0/attention/self/query/bias/adam_m" input: "add_25" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_226" op: "Assign" input: "bert/encoder/layer_0/attention/self/query/bias/adam_v" input: "add_26" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_m" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_v" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_43/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_43" op: "Mul" input: "Mul_43/x" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_44/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_44" op: "Mul" input: "Mul_44/x" input: "clip_by_global_norm/clip_by_global_norm/_7" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_28" op: "Add" input: "Mul_43" input: "Mul_44" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_45/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_45" op: "Mul" input: "Mul_45/x" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_7" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_7" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_46/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_46" op: "Mul" input: "Mul_46/x" input: "Square_7" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_29" op: "Add" input: "Mul_45" input: "Mul_46" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_7" op: "Sqrt" input: "add_29" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_30/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_30" op: "Add" input: "Sqrt_7" input: "add_30/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_8" op: "RealDiv" input: "add_28" input: "add_30" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_47/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_47" op: "Mul" input: "mul_47/x" input: "bert/encoder/layer_0/attention/self/key/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_31" op: "Add" input: "truediv_8" input: "mul_47" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_48" op: "Mul" input: "add_2" input: "add_31" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_8" op: "Sub" input: "bert/encoder/layer_0/attention/self/key/kernel/read" input: "mul_48" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_227" op: "Assign" input: "bert/encoder/layer_0/attention/self/key/kernel" input: "sub_8" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_228" op: "Assign" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_m" input: "add_28" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_229" op: "Assign" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_v" input: "add_29" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/attention/self/key/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/self/key/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/attention/self/key/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_0/attention/self/key/bias/adam_m" input: "bert/encoder/layer_0/attention/self/key/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/attention/self/key/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_0/attention/self/key/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/self/key/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/self/key/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/attention/self/key/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_0/attention/self/key/bias/adam_v" input: "bert/encoder/layer_0/attention/self/key/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/attention/self/key/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_0/attention/self/key/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_49/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_49" op: "Mul" input: "Mul_49/x" input: "bert/encoder/layer_0/attention/self/key/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_50/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_50" op: "Mul" input: "Mul_50/x" input: "clip_by_global_norm/clip_by_global_norm/_8" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_32" op: "Add" input: "Mul_49" input: "Mul_50" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_51/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_51" op: "Mul" input: "Mul_51/x" input: "bert/encoder/layer_0/attention/self/key/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_8" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_8" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_52/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_52" op: "Mul" input: "Mul_52/x" input: "Square_8" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_33" op: "Add" input: "Mul_51" input: "Mul_52" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_8" op: "Sqrt" input: "add_33" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_34/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_34" op: "Add" input: "Sqrt_8" input: "add_34/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_9" op: "RealDiv" input: "add_32" input: "add_34" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_53" op: "Mul" input: "add_2" input: "truediv_9" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_9" op: "Sub" input: "bert/encoder/layer_0/attention/self/key/bias/read" input: "mul_53" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_230" op: "Assign" input: "bert/encoder/layer_0/attention/self/key/bias" input: "sub_9" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_231" op: "Assign" input: "bert/encoder/layer_0/attention/self/key/bias/adam_m" input: "add_32" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_232" op: "Assign" input: "bert/encoder/layer_0/attention/self/key/bias/adam_v" input: "add_33" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_m" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_v" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_54/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_54" op: "Mul" input: "Mul_54/x" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_55/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_55" op: "Mul" input: "Mul_55/x" input: "clip_by_global_norm/clip_by_global_norm/_9" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_35" op: "Add" input: "Mul_54" input: "Mul_55" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_56/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_56" op: "Mul" input: "Mul_56/x" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_9" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_9" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_57/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_57" op: "Mul" input: "Mul_57/x" input: "Square_9" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_36" op: "Add" input: "Mul_56" input: "Mul_57" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_9" op: "Sqrt" input: "add_36" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_37/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_37" op: "Add" input: "Sqrt_9" input: "add_37/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_10" op: "RealDiv" input: "add_35" input: "add_37" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_58/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_58" op: "Mul" input: "mul_58/x" input: "bert/encoder/layer_0/attention/self/value/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_38" op: "Add" input: "truediv_10" input: "mul_58" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_59" op: "Mul" input: "add_2" input: "add_38" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_10" op: "Sub" input: "bert/encoder/layer_0/attention/self/value/kernel/read" input: "mul_59" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_233" op: "Assign" input: "bert/encoder/layer_0/attention/self/value/kernel" input: "sub_10" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_234" op: "Assign" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_m" input: "add_35" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_235" op: "Assign" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_v" input: "add_36" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/attention/self/value/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/self/value/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/attention/self/value/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_0/attention/self/value/bias/adam_m" input: "bert/encoder/layer_0/attention/self/value/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/attention/self/value/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_0/attention/self/value/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/self/value/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/self/value/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/attention/self/value/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_0/attention/self/value/bias/adam_v" input: "bert/encoder/layer_0/attention/self/value/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/attention/self/value/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_0/attention/self/value/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_60/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_60" op: "Mul" input: "Mul_60/x" input: "bert/encoder/layer_0/attention/self/value/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_61/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_61" op: "Mul" input: "Mul_61/x" input: "clip_by_global_norm/clip_by_global_norm/_10" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_39" op: "Add" input: "Mul_60" input: "Mul_61" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_62/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_62" op: "Mul" input: "Mul_62/x" input: "bert/encoder/layer_0/attention/self/value/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_10" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_10" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_63/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_63" op: "Mul" input: "Mul_63/x" input: "Square_10" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_40" op: "Add" input: "Mul_62" input: "Mul_63" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_10" op: "Sqrt" input: "add_40" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_41/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_41" op: "Add" input: "Sqrt_10" input: "add_41/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_11" op: "RealDiv" input: "add_39" input: "add_41" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_64" op: "Mul" input: "add_2" input: "truediv_11" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_11" op: "Sub" input: "bert/encoder/layer_0/attention/self/value/bias/read" input: "mul_64" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_236" op: "Assign" input: "bert/encoder/layer_0/attention/self/value/bias" input: "sub_11" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_237" op: "Assign" input: "bert/encoder/layer_0/attention/self/value/bias/adam_m" input: "add_39" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_238" op: "Assign" input: "bert/encoder/layer_0/attention/self/value/bias/adam_v" input: "add_40" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_65/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_65" op: "Mul" input: "Mul_65/x" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_66/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_66" op: "Mul" input: "Mul_66/x" input: "clip_by_global_norm/clip_by_global_norm/_11" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_42" op: "Add" input: "Mul_65" input: "Mul_66" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_67/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_67" op: "Mul" input: "Mul_67/x" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_11" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_11" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_68/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_68" op: "Mul" input: "Mul_68/x" input: "Square_11" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_43" op: "Add" input: "Mul_67" input: "Mul_68" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_11" op: "Sqrt" input: "add_43" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_44/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_44" op: "Add" input: "Sqrt_11" input: "add_44/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_12" op: "RealDiv" input: "add_42" input: "add_44" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_69/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_69" op: "Mul" input: "mul_69/x" input: "bert/encoder/layer_0/attention/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_45" op: "Add" input: "truediv_12" input: "mul_69" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_70" op: "Mul" input: "add_2" input: "add_45" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_12" op: "Sub" input: "bert/encoder/layer_0/attention/output/dense/kernel/read" input: "mul_70" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_239" op: "Assign" input: "bert/encoder/layer_0/attention/output/dense/kernel" input: "sub_12" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_240" op: "Assign" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m" input: "add_42" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_241" op: "Assign" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v" input: "add_43" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/attention/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/output/dense/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/attention/output/dense/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_0/attention/output/dense/bias/adam_m" input: "bert/encoder/layer_0/attention/output/dense/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/attention/output/dense/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_0/attention/output/dense/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/output/dense/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/attention/output/dense/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_0/attention/output/dense/bias/adam_v" input: "bert/encoder/layer_0/attention/output/dense/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/attention/output/dense/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_0/attention/output/dense/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_71/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_71" op: "Mul" input: "Mul_71/x" input: "bert/encoder/layer_0/attention/output/dense/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_72/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_72" op: "Mul" input: "Mul_72/x" input: "clip_by_global_norm/clip_by_global_norm/_12" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_46" op: "Add" input: "Mul_71" input: "Mul_72" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_73/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_73" op: "Mul" input: "Mul_73/x" input: "bert/encoder/layer_0/attention/output/dense/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_12" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_12" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_74/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_74" op: "Mul" input: "Mul_74/x" input: "Square_12" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_47" op: "Add" input: "Mul_73" input: "Mul_74" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_12" op: "Sqrt" input: "add_47" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_48/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_48" op: "Add" input: "Sqrt_12" input: "add_48/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_13" op: "RealDiv" input: "add_46" input: "add_48" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_75" op: "Mul" input: "add_2" input: "truediv_13" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_13" op: "Sub" input: "bert/encoder/layer_0/attention/output/dense/bias/read" input: "mul_75" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_242" op: "Assign" input: "bert/encoder/layer_0/attention/output/dense/bias" input: "sub_13" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_243" op: "Assign" input: "bert/encoder/layer_0/attention/output/dense/bias/adam_m" input: "add_46" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_244" op: "Assign" input: "bert/encoder/layer_0/attention/output/dense/bias/adam_v" input: "add_47" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_m/read" op: "Identity" input: "bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_v/read" op: "Identity" input: "bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_76/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_76" op: "Mul" input: "Mul_76/x" input: "bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_77/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_77" op: "Mul" input: "Mul_77/x" input: "clip_by_global_norm/clip_by_global_norm/_13" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_49" op: "Add" input: "Mul_76" input: "Mul_77" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_78/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_78" op: "Mul" input: "Mul_78/x" input: "bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_13" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_13" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_79/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_79" op: "Mul" input: "Mul_79/x" input: "Square_13" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_50" op: "Add" input: "Mul_78" input: "Mul_79" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_13" op: "Sqrt" input: "add_50" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_51/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_51" op: "Add" input: "Sqrt_13" input: "add_51/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_14" op: "RealDiv" input: "add_49" input: "add_51" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_80" op: "Mul" input: "add_2" input: "truediv_14" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_14" op: "Sub" input: "bert/encoder/layer_0/attention/output/LayerNorm/beta/read" input: "mul_80" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_245" op: "Assign" input: "bert/encoder/layer_0/attention/output/LayerNorm/beta" input: "sub_14" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_246" op: "Assign" input: "bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_m" input: "add_49" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_247" op: "Assign" input: "bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_v" input: "add_50" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m/read" op: "Identity" input: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v/read" op: "Identity" input: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_81/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_81" op: "Mul" input: "Mul_81/x" input: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_82/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_82" op: "Mul" input: "Mul_82/x" input: "clip_by_global_norm/clip_by_global_norm/_14" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_52" op: "Add" input: "Mul_81" input: "Mul_82" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_83/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_83" op: "Mul" input: "Mul_83/x" input: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_14" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_14" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_84/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_84" op: "Mul" input: "Mul_84/x" input: "Square_14" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_53" op: "Add" input: "Mul_83" input: "Mul_84" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_14" op: "Sqrt" input: "add_53" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_54/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_54" op: "Add" input: "Sqrt_14" input: "add_54/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_15" op: "RealDiv" input: "add_52" input: "add_54" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_85" op: "Mul" input: "add_2" input: "truediv_15" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_15" op: "Sub" input: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/read" input: "mul_85" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_248" op: "Assign" input: "bert/encoder/layer_0/attention/output/LayerNorm/gamma" input: "sub_15" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_249" op: "Assign" input: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m" input: "add_52" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_250" op: "Assign" input: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v" input: "add_53" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_86/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_86" op: "Mul" input: "Mul_86/x" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_87/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_87" op: "Mul" input: "Mul_87/x" input: "clip_by_global_norm/clip_by_global_norm/_15" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_55" op: "Add" input: "Mul_86" input: "Mul_87" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_88/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_88" op: "Mul" input: "Mul_88/x" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Square_15" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_15" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_89/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_89" op: "Mul" input: "Mul_89/x" input: "Square_15" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_56" op: "Add" input: "Mul_88" input: "Mul_89" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Sqrt_15" op: "Sqrt" input: "add_56" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_57/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_57" op: "Add" input: "Sqrt_15" input: "add_57/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "truediv_16" op: "RealDiv" input: "add_55" input: "add_57" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "mul_90/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_90" op: "Mul" input: "mul_90/x" input: "bert/encoder/layer_0/intermediate/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_58" op: "Add" input: "truediv_16" input: "mul_90" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "mul_91" op: "Mul" input: "add_2" input: "add_58" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "sub_16" op: "Sub" input: "bert/encoder/layer_0/intermediate/dense/kernel/read" input: "mul_91" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Assign_251" op: "Assign" input: "bert/encoder/layer_0/intermediate/dense/kernel" input: "sub_16" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_252" op: "Assign" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m" input: "add_55" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_253" op: "Assign" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v" input: "add_56" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_0/intermediate/dense/bias/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/intermediate/dense/bias/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_0/intermediate/dense/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/intermediate/dense/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_m" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/intermediate/dense/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_0/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_0/intermediate/dense/bias/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/intermediate/dense/bias/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_0/intermediate/dense/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/intermediate/dense/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_v" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/intermediate/dense/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_92/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_92" op: "Mul" input: "Mul_92/x" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_93/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_93" op: "Mul" input: "Mul_93/x" input: "clip_by_global_norm/clip_by_global_norm/_16" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_59" op: "Add" input: "Mul_92" input: "Mul_93" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_94/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_94" op: "Mul" input: "Mul_94/x" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Square_16" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_16" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_95/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_95" op: "Mul" input: "Mul_95/x" input: "Square_16" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_60" op: "Add" input: "Mul_94" input: "Mul_95" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Sqrt_16" op: "Sqrt" input: "add_60" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_61/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_61" op: "Add" input: "Sqrt_16" input: "add_61/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "truediv_17" op: "RealDiv" input: "add_59" input: "add_61" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "mul_96" op: "Mul" input: "add_2" input: "truediv_17" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "sub_17" op: "Sub" input: "bert/encoder/layer_0/intermediate/dense/bias/read" input: "mul_96" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Assign_254" op: "Assign" input: "bert/encoder/layer_0/intermediate/dense/bias" input: "sub_17" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_255" op: "Assign" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_m" input: "add_59" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_256" op: "Assign" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_v" input: "add_60" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_0/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_0/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_0/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_0/output/dense/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/output/dense/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_0/output/dense/kernel/adam_m" input: "bert/encoder/layer_0/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/output/dense/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_0/output/dense/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_0/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_0/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_0/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_0/output/dense/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/output/dense/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_0/output/dense/kernel/adam_v" input: "bert/encoder/layer_0/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/output/dense/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_0/output/dense/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_97/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_97" op: "Mul" input: "Mul_97/x" input: "bert/encoder/layer_0/output/dense/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_98/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_98" op: "Mul" input: "Mul_98/x" input: "clip_by_global_norm/clip_by_global_norm/_17" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_62" op: "Add" input: "Mul_97" input: "Mul_98" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_99/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_99" op: "Mul" input: "Mul_99/x" input: "bert/encoder/layer_0/output/dense/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Square_17" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_17" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_100/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_100" op: "Mul" input: "Mul_100/x" input: "Square_17" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_63" op: "Add" input: "Mul_99" input: "Mul_100" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Sqrt_17" op: "Sqrt" input: "add_63" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_64/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_64" op: "Add" input: "Sqrt_17" input: "add_64/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "truediv_18" op: "RealDiv" input: "add_62" input: "add_64" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "mul_101/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_101" op: "Mul" input: "mul_101/x" input: "bert/encoder/layer_0/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_65" op: "Add" input: "truediv_18" input: "mul_101" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "mul_102" op: "Mul" input: "add_2" input: "add_65" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "sub_18" op: "Sub" input: "bert/encoder/layer_0/output/dense/kernel/read" input: "mul_102" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Assign_257" op: "Assign" input: "bert/encoder/layer_0/output/dense/kernel" input: "sub_18" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_258" op: "Assign" input: "bert/encoder/layer_0/output/dense/kernel/adam_m" input: "add_62" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_259" op: "Assign" input: "bert/encoder/layer_0/output/dense/kernel/adam_v" input: "add_63" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/output/dense/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/output/dense/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_0/output/dense/bias/adam_m" input: "bert/encoder/layer_0/output/dense/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/output/dense/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_0/output/dense/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/output/dense/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/output/dense/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_0/output/dense/bias/adam_v" input: "bert/encoder/layer_0/output/dense/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/output/dense/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_0/output/dense/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_103/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_103" op: "Mul" input: "Mul_103/x" input: "bert/encoder/layer_0/output/dense/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_104/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_104" op: "Mul" input: "Mul_104/x" input: "clip_by_global_norm/clip_by_global_norm/_18" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_66" op: "Add" input: "Mul_103" input: "Mul_104" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_105/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_105" op: "Mul" input: "Mul_105/x" input: "bert/encoder/layer_0/output/dense/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_18" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_18" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_106/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_106" op: "Mul" input: "Mul_106/x" input: "Square_18" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_67" op: "Add" input: "Mul_105" input: "Mul_106" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_18" op: "Sqrt" input: "add_67" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_68/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_68" op: "Add" input: "Sqrt_18" input: "add_68/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_19" op: "RealDiv" input: "add_66" input: "add_68" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_107" op: "Mul" input: "add_2" input: "truediv_19" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_19" op: "Sub" input: "bert/encoder/layer_0/output/dense/bias/read" input: "mul_107" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_260" op: "Assign" input: "bert/encoder/layer_0/output/dense/bias" input: "sub_19" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_261" op: "Assign" input: "bert/encoder/layer_0/output/dense/bias/adam_m" input: "add_66" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_262" op: "Assign" input: "bert/encoder/layer_0/output/dense/bias/adam_v" input: "add_67" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/output/LayerNorm/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/output/LayerNorm/beta/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/output/LayerNorm/beta/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_0/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_0/output/LayerNorm/beta/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/output/LayerNorm/beta/adam_m/read" op: "Identity" input: "bert/encoder/layer_0/output/LayerNorm/beta/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/output/LayerNorm/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/output/LayerNorm/beta/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/output/LayerNorm/beta/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_0/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_0/output/LayerNorm/beta/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/output/LayerNorm/beta/adam_v/read" op: "Identity" input: "bert/encoder/layer_0/output/LayerNorm/beta/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_108/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_108" op: "Mul" input: "Mul_108/x" input: "bert/encoder/layer_0/output/LayerNorm/beta/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_109/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_109" op: "Mul" input: "Mul_109/x" input: "clip_by_global_norm/clip_by_global_norm/_19" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_69" op: "Add" input: "Mul_108" input: "Mul_109" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_110/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_110" op: "Mul" input: "Mul_110/x" input: "bert/encoder/layer_0/output/LayerNorm/beta/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_19" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_19" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_111/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_111" op: "Mul" input: "Mul_111/x" input: "Square_19" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_70" op: "Add" input: "Mul_110" input: "Mul_111" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_19" op: "Sqrt" input: "add_70" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_71/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_71" op: "Add" input: "Sqrt_19" input: "add_71/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_20" op: "RealDiv" input: "add_69" input: "add_71" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_112" op: "Mul" input: "add_2" input: "truediv_20" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_20" op: "Sub" input: "bert/encoder/layer_0/output/LayerNorm/beta/read" input: "mul_112" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_263" op: "Assign" input: "bert/encoder/layer_0/output/LayerNorm/beta" input: "sub_20" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_264" op: "Assign" input: "bert/encoder/layer_0/output/LayerNorm/beta/adam_m" input: "add_69" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_265" op: "Assign" input: "bert/encoder/layer_0/output/LayerNorm/beta/adam_v" input: "add_70" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/output/LayerNorm/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/output/LayerNorm/gamma/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/output/LayerNorm/gamma/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_0/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_0/output/LayerNorm/gamma/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/output/LayerNorm/gamma/adam_m/read" op: "Identity" input: "bert/encoder/layer_0/output/LayerNorm/gamma/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/output/LayerNorm/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/output/LayerNorm/gamma/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_0/output/LayerNorm/gamma/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_0/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_0/output/LayerNorm/gamma/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_0/output/LayerNorm/gamma/adam_v/read" op: "Identity" input: "bert/encoder/layer_0/output/LayerNorm/gamma/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_113/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_113" op: "Mul" input: "Mul_113/x" input: "bert/encoder/layer_0/output/LayerNorm/gamma/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_114/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_114" op: "Mul" input: "Mul_114/x" input: "clip_by_global_norm/clip_by_global_norm/_20" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_72" op: "Add" input: "Mul_113" input: "Mul_114" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_115/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_115" op: "Mul" input: "Mul_115/x" input: "bert/encoder/layer_0/output/LayerNorm/gamma/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_20" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_20" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_116/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_116" op: "Mul" input: "Mul_116/x" input: "Square_20" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_73" op: "Add" input: "Mul_115" input: "Mul_116" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_20" op: "Sqrt" input: "add_73" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_74/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_74" op: "Add" input: "Sqrt_20" input: "add_74/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_21" op: "RealDiv" input: "add_72" input: "add_74" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_117" op: "Mul" input: "add_2" input: "truediv_21" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_21" op: "Sub" input: "bert/encoder/layer_0/output/LayerNorm/gamma/read" input: "mul_117" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_266" op: "Assign" input: "bert/encoder/layer_0/output/LayerNorm/gamma" input: "sub_21" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_267" op: "Assign" input: "bert/encoder/layer_0/output/LayerNorm/gamma/adam_m" input: "add_72" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_268" op: "Assign" input: "bert/encoder/layer_0/output/LayerNorm/gamma/adam_v" input: "add_73" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_m" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_v" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_118/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_118" op: "Mul" input: "Mul_118/x" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_119/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_119" op: "Mul" input: "Mul_119/x" input: "clip_by_global_norm/clip_by_global_norm/_21" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_75" op: "Add" input: "Mul_118" input: "Mul_119" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_120/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_120" op: "Mul" input: "Mul_120/x" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_21" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_21" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_121/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_121" op: "Mul" input: "Mul_121/x" input: "Square_21" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_76" op: "Add" input: "Mul_120" input: "Mul_121" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_21" op: "Sqrt" input: "add_76" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_77/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_77" op: "Add" input: "Sqrt_21" input: "add_77/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_22" op: "RealDiv" input: "add_75" input: "add_77" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_122/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_122" op: "Mul" input: "mul_122/x" input: "bert/encoder/layer_1/attention/self/query/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_78" op: "Add" input: "truediv_22" input: "mul_122" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_123" op: "Mul" input: "add_2" input: "add_78" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_22" op: "Sub" input: "bert/encoder/layer_1/attention/self/query/kernel/read" input: "mul_123" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_269" op: "Assign" input: "bert/encoder/layer_1/attention/self/query/kernel" input: "sub_22" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_270" op: "Assign" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_m" input: "add_75" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_271" op: "Assign" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_v" input: "add_76" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/attention/self/query/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/self/query/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/attention/self/query/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_1/attention/self/query/bias/adam_m" input: "bert/encoder/layer_1/attention/self/query/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/attention/self/query/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_1/attention/self/query/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/self/query/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/self/query/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/attention/self/query/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_1/attention/self/query/bias/adam_v" input: "bert/encoder/layer_1/attention/self/query/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/attention/self/query/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_1/attention/self/query/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_124/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_124" op: "Mul" input: "Mul_124/x" input: "bert/encoder/layer_1/attention/self/query/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_125/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_125" op: "Mul" input: "Mul_125/x" input: "clip_by_global_norm/clip_by_global_norm/_22" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_79" op: "Add" input: "Mul_124" input: "Mul_125" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_126/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_126" op: "Mul" input: "Mul_126/x" input: "bert/encoder/layer_1/attention/self/query/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_22" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_22" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_127/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_127" op: "Mul" input: "Mul_127/x" input: "Square_22" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_80" op: "Add" input: "Mul_126" input: "Mul_127" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_22" op: "Sqrt" input: "add_80" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_81/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_81" op: "Add" input: "Sqrt_22" input: "add_81/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_23" op: "RealDiv" input: "add_79" input: "add_81" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_128" op: "Mul" input: "add_2" input: "truediv_23" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_23" op: "Sub" input: "bert/encoder/layer_1/attention/self/query/bias/read" input: "mul_128" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_272" op: "Assign" input: "bert/encoder/layer_1/attention/self/query/bias" input: "sub_23" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_273" op: "Assign" input: "bert/encoder/layer_1/attention/self/query/bias/adam_m" input: "add_79" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_274" op: "Assign" input: "bert/encoder/layer_1/attention/self/query/bias/adam_v" input: "add_80" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_m" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_v" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_129/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_129" op: "Mul" input: "Mul_129/x" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_130/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_130" op: "Mul" input: "Mul_130/x" input: "clip_by_global_norm/clip_by_global_norm/_23" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_82" op: "Add" input: "Mul_129" input: "Mul_130" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_131/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_131" op: "Mul" input: "Mul_131/x" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_23" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_23" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_132/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_132" op: "Mul" input: "Mul_132/x" input: "Square_23" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_83" op: "Add" input: "Mul_131" input: "Mul_132" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_23" op: "Sqrt" input: "add_83" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_84/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_84" op: "Add" input: "Sqrt_23" input: "add_84/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_24" op: "RealDiv" input: "add_82" input: "add_84" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_133/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_133" op: "Mul" input: "mul_133/x" input: "bert/encoder/layer_1/attention/self/key/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_85" op: "Add" input: "truediv_24" input: "mul_133" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_134" op: "Mul" input: "add_2" input: "add_85" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_24" op: "Sub" input: "bert/encoder/layer_1/attention/self/key/kernel/read" input: "mul_134" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_275" op: "Assign" input: "bert/encoder/layer_1/attention/self/key/kernel" input: "sub_24" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_276" op: "Assign" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_m" input: "add_82" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_277" op: "Assign" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_v" input: "add_83" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/attention/self/key/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/self/key/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/attention/self/key/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_1/attention/self/key/bias/adam_m" input: "bert/encoder/layer_1/attention/self/key/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/attention/self/key/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_1/attention/self/key/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/self/key/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/self/key/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/attention/self/key/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_1/attention/self/key/bias/adam_v" input: "bert/encoder/layer_1/attention/self/key/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/attention/self/key/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_1/attention/self/key/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_135/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_135" op: "Mul" input: "Mul_135/x" input: "bert/encoder/layer_1/attention/self/key/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_136/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_136" op: "Mul" input: "Mul_136/x" input: "clip_by_global_norm/clip_by_global_norm/_24" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_86" op: "Add" input: "Mul_135" input: "Mul_136" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_137/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_137" op: "Mul" input: "Mul_137/x" input: "bert/encoder/layer_1/attention/self/key/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_24" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_24" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_138/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_138" op: "Mul" input: "Mul_138/x" input: "Square_24" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_87" op: "Add" input: "Mul_137" input: "Mul_138" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_24" op: "Sqrt" input: "add_87" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_88/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_88" op: "Add" input: "Sqrt_24" input: "add_88/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_25" op: "RealDiv" input: "add_86" input: "add_88" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_139" op: "Mul" input: "add_2" input: "truediv_25" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_25" op: "Sub" input: "bert/encoder/layer_1/attention/self/key/bias/read" input: "mul_139" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_278" op: "Assign" input: "bert/encoder/layer_1/attention/self/key/bias" input: "sub_25" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_279" op: "Assign" input: "bert/encoder/layer_1/attention/self/key/bias/adam_m" input: "add_86" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_280" op: "Assign" input: "bert/encoder/layer_1/attention/self/key/bias/adam_v" input: "add_87" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_m" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_v" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_140/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_140" op: "Mul" input: "Mul_140/x" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_141/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_141" op: "Mul" input: "Mul_141/x" input: "clip_by_global_norm/clip_by_global_norm/_25" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_89" op: "Add" input: "Mul_140" input: "Mul_141" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_142/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_142" op: "Mul" input: "Mul_142/x" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_25" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_25" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_143/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_143" op: "Mul" input: "Mul_143/x" input: "Square_25" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_90" op: "Add" input: "Mul_142" input: "Mul_143" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_25" op: "Sqrt" input: "add_90" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_91/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_91" op: "Add" input: "Sqrt_25" input: "add_91/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_26" op: "RealDiv" input: "add_89" input: "add_91" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_144/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_144" op: "Mul" input: "mul_144/x" input: "bert/encoder/layer_1/attention/self/value/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_92" op: "Add" input: "truediv_26" input: "mul_144" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_145" op: "Mul" input: "add_2" input: "add_92" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_26" op: "Sub" input: "bert/encoder/layer_1/attention/self/value/kernel/read" input: "mul_145" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_281" op: "Assign" input: "bert/encoder/layer_1/attention/self/value/kernel" input: "sub_26" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_282" op: "Assign" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_m" input: "add_89" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_283" op: "Assign" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_v" input: "add_90" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/attention/self/value/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/self/value/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/attention/self/value/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_1/attention/self/value/bias/adam_m" input: "bert/encoder/layer_1/attention/self/value/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/attention/self/value/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_1/attention/self/value/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/self/value/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/self/value/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/attention/self/value/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_1/attention/self/value/bias/adam_v" input: "bert/encoder/layer_1/attention/self/value/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/attention/self/value/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_1/attention/self/value/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_146/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_146" op: "Mul" input: "Mul_146/x" input: "bert/encoder/layer_1/attention/self/value/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_147/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_147" op: "Mul" input: "Mul_147/x" input: "clip_by_global_norm/clip_by_global_norm/_26" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_93" op: "Add" input: "Mul_146" input: "Mul_147" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_148/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_148" op: "Mul" input: "Mul_148/x" input: "bert/encoder/layer_1/attention/self/value/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_26" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_26" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_149/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_149" op: "Mul" input: "Mul_149/x" input: "Square_26" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_94" op: "Add" input: "Mul_148" input: "Mul_149" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_26" op: "Sqrt" input: "add_94" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_95/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_95" op: "Add" input: "Sqrt_26" input: "add_95/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_27" op: "RealDiv" input: "add_93" input: "add_95" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_150" op: "Mul" input: "add_2" input: "truediv_27" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_27" op: "Sub" input: "bert/encoder/layer_1/attention/self/value/bias/read" input: "mul_150" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_284" op: "Assign" input: "bert/encoder/layer_1/attention/self/value/bias" input: "sub_27" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_285" op: "Assign" input: "bert/encoder/layer_1/attention/self/value/bias/adam_m" input: "add_93" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_286" op: "Assign" input: "bert/encoder/layer_1/attention/self/value/bias/adam_v" input: "add_94" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_151/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_151" op: "Mul" input: "Mul_151/x" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_152/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_152" op: "Mul" input: "Mul_152/x" input: "clip_by_global_norm/clip_by_global_norm/_27" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_96" op: "Add" input: "Mul_151" input: "Mul_152" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_153/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_153" op: "Mul" input: "Mul_153/x" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_27" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_27" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_154/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_154" op: "Mul" input: "Mul_154/x" input: "Square_27" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_97" op: "Add" input: "Mul_153" input: "Mul_154" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_27" op: "Sqrt" input: "add_97" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_98/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_98" op: "Add" input: "Sqrt_27" input: "add_98/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_28" op: "RealDiv" input: "add_96" input: "add_98" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_155/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_155" op: "Mul" input: "mul_155/x" input: "bert/encoder/layer_1/attention/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_99" op: "Add" input: "truediv_28" input: "mul_155" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_156" op: "Mul" input: "add_2" input: "add_99" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_28" op: "Sub" input: "bert/encoder/layer_1/attention/output/dense/kernel/read" input: "mul_156" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_287" op: "Assign" input: "bert/encoder/layer_1/attention/output/dense/kernel" input: "sub_28" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_288" op: "Assign" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m" input: "add_96" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_289" op: "Assign" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v" input: "add_97" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/attention/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/output/dense/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/attention/output/dense/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_1/attention/output/dense/bias/adam_m" input: "bert/encoder/layer_1/attention/output/dense/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/attention/output/dense/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_1/attention/output/dense/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/output/dense/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/attention/output/dense/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_1/attention/output/dense/bias/adam_v" input: "bert/encoder/layer_1/attention/output/dense/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/attention/output/dense/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_1/attention/output/dense/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_157/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_157" op: "Mul" input: "Mul_157/x" input: "bert/encoder/layer_1/attention/output/dense/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_158/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_158" op: "Mul" input: "Mul_158/x" input: "clip_by_global_norm/clip_by_global_norm/_28" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_100" op: "Add" input: "Mul_157" input: "Mul_158" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_159/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_159" op: "Mul" input: "Mul_159/x" input: "bert/encoder/layer_1/attention/output/dense/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_28" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_28" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_160/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_160" op: "Mul" input: "Mul_160/x" input: "Square_28" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_101" op: "Add" input: "Mul_159" input: "Mul_160" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_28" op: "Sqrt" input: "add_101" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_102/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_102" op: "Add" input: "Sqrt_28" input: "add_102/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_29" op: "RealDiv" input: "add_100" input: "add_102" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_161" op: "Mul" input: "add_2" input: "truediv_29" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_29" op: "Sub" input: "bert/encoder/layer_1/attention/output/dense/bias/read" input: "mul_161" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_290" op: "Assign" input: "bert/encoder/layer_1/attention/output/dense/bias" input: "sub_29" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_291" op: "Assign" input: "bert/encoder/layer_1/attention/output/dense/bias/adam_m" input: "add_100" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_292" op: "Assign" input: "bert/encoder/layer_1/attention/output/dense/bias/adam_v" input: "add_101" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_m/read" op: "Identity" input: "bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_v/read" op: "Identity" input: "bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_162/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_162" op: "Mul" input: "Mul_162/x" input: "bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_163/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_163" op: "Mul" input: "Mul_163/x" input: "clip_by_global_norm/clip_by_global_norm/_29" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_103" op: "Add" input: "Mul_162" input: "Mul_163" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_164/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_164" op: "Mul" input: "Mul_164/x" input: "bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_29" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_29" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_165/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_165" op: "Mul" input: "Mul_165/x" input: "Square_29" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_104" op: "Add" input: "Mul_164" input: "Mul_165" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_29" op: "Sqrt" input: "add_104" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_105/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_105" op: "Add" input: "Sqrt_29" input: "add_105/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_30" op: "RealDiv" input: "add_103" input: "add_105" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_166" op: "Mul" input: "add_2" input: "truediv_30" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_30" op: "Sub" input: "bert/encoder/layer_1/attention/output/LayerNorm/beta/read" input: "mul_166" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_293" op: "Assign" input: "bert/encoder/layer_1/attention/output/LayerNorm/beta" input: "sub_30" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_294" op: "Assign" input: "bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_m" input: "add_103" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_295" op: "Assign" input: "bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_v" input: "add_104" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m/read" op: "Identity" input: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v/read" op: "Identity" input: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_167/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_167" op: "Mul" input: "Mul_167/x" input: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_168/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_168" op: "Mul" input: "Mul_168/x" input: "clip_by_global_norm/clip_by_global_norm/_30" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_106" op: "Add" input: "Mul_167" input: "Mul_168" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_169/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_169" op: "Mul" input: "Mul_169/x" input: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_30" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_30" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_170/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_170" op: "Mul" input: "Mul_170/x" input: "Square_30" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_107" op: "Add" input: "Mul_169" input: "Mul_170" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_30" op: "Sqrt" input: "add_107" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_108/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_108" op: "Add" input: "Sqrt_30" input: "add_108/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_31" op: "RealDiv" input: "add_106" input: "add_108" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_171" op: "Mul" input: "add_2" input: "truediv_31" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_31" op: "Sub" input: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/read" input: "mul_171" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_296" op: "Assign" input: "bert/encoder/layer_1/attention/output/LayerNorm/gamma" input: "sub_31" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_297" op: "Assign" input: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m" input: "add_106" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_298" op: "Assign" input: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v" input: "add_107" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_172/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_172" op: "Mul" input: "Mul_172/x" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_173/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_173" op: "Mul" input: "Mul_173/x" input: "clip_by_global_norm/clip_by_global_norm/_31" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_109" op: "Add" input: "Mul_172" input: "Mul_173" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_174/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_174" op: "Mul" input: "Mul_174/x" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Square_31" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_31" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_175/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_175" op: "Mul" input: "Mul_175/x" input: "Square_31" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_110" op: "Add" input: "Mul_174" input: "Mul_175" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Sqrt_31" op: "Sqrt" input: "add_110" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_111/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_111" op: "Add" input: "Sqrt_31" input: "add_111/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "truediv_32" op: "RealDiv" input: "add_109" input: "add_111" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "mul_176/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_176" op: "Mul" input: "mul_176/x" input: "bert/encoder/layer_1/intermediate/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_112" op: "Add" input: "truediv_32" input: "mul_176" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "mul_177" op: "Mul" input: "add_2" input: "add_112" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "sub_32" op: "Sub" input: "bert/encoder/layer_1/intermediate/dense/kernel/read" input: "mul_177" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Assign_299" op: "Assign" input: "bert/encoder/layer_1/intermediate/dense/kernel" input: "sub_32" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_300" op: "Assign" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m" input: "add_109" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_301" op: "Assign" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v" input: "add_110" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_1/intermediate/dense/bias/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/intermediate/dense/bias/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_1/intermediate/dense/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/intermediate/dense/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_m" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/intermediate/dense/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_1/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_1/intermediate/dense/bias/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/intermediate/dense/bias/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_1/intermediate/dense/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/intermediate/dense/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_v" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/intermediate/dense/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_178/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_178" op: "Mul" input: "Mul_178/x" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_179/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_179" op: "Mul" input: "Mul_179/x" input: "clip_by_global_norm/clip_by_global_norm/_32" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_113" op: "Add" input: "Mul_178" input: "Mul_179" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_180/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_180" op: "Mul" input: "Mul_180/x" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Square_32" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_32" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_181/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_181" op: "Mul" input: "Mul_181/x" input: "Square_32" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_114" op: "Add" input: "Mul_180" input: "Mul_181" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Sqrt_32" op: "Sqrt" input: "add_114" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_115/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_115" op: "Add" input: "Sqrt_32" input: "add_115/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "truediv_33" op: "RealDiv" input: "add_113" input: "add_115" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "mul_182" op: "Mul" input: "add_2" input: "truediv_33" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "sub_33" op: "Sub" input: "bert/encoder/layer_1/intermediate/dense/bias/read" input: "mul_182" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Assign_302" op: "Assign" input: "bert/encoder/layer_1/intermediate/dense/bias" input: "sub_33" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_303" op: "Assign" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_m" input: "add_113" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_304" op: "Assign" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_v" input: "add_114" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_1/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_1/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_1/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_1/output/dense/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/output/dense/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_1/output/dense/kernel/adam_m" input: "bert/encoder/layer_1/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/output/dense/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_1/output/dense/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_1/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_1/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_1/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_1/output/dense/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/output/dense/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_1/output/dense/kernel/adam_v" input: "bert/encoder/layer_1/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/output/dense/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_1/output/dense/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_183/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_183" op: "Mul" input: "Mul_183/x" input: "bert/encoder/layer_1/output/dense/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_184/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_184" op: "Mul" input: "Mul_184/x" input: "clip_by_global_norm/clip_by_global_norm/_33" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_116" op: "Add" input: "Mul_183" input: "Mul_184" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_185/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_185" op: "Mul" input: "Mul_185/x" input: "bert/encoder/layer_1/output/dense/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Square_33" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_33" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_186/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_186" op: "Mul" input: "Mul_186/x" input: "Square_33" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_117" op: "Add" input: "Mul_185" input: "Mul_186" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Sqrt_33" op: "Sqrt" input: "add_117" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_118/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_118" op: "Add" input: "Sqrt_33" input: "add_118/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "truediv_34" op: "RealDiv" input: "add_116" input: "add_118" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "mul_187/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_187" op: "Mul" input: "mul_187/x" input: "bert/encoder/layer_1/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_119" op: "Add" input: "truediv_34" input: "mul_187" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "mul_188" op: "Mul" input: "add_2" input: "add_119" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "sub_34" op: "Sub" input: "bert/encoder/layer_1/output/dense/kernel/read" input: "mul_188" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Assign_305" op: "Assign" input: "bert/encoder/layer_1/output/dense/kernel" input: "sub_34" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_306" op: "Assign" input: "bert/encoder/layer_1/output/dense/kernel/adam_m" input: "add_116" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_307" op: "Assign" input: "bert/encoder/layer_1/output/dense/kernel/adam_v" input: "add_117" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/output/dense/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/output/dense/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_1/output/dense/bias/adam_m" input: "bert/encoder/layer_1/output/dense/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/output/dense/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_1/output/dense/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/output/dense/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/output/dense/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_1/output/dense/bias/adam_v" input: "bert/encoder/layer_1/output/dense/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/output/dense/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_1/output/dense/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_189/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_189" op: "Mul" input: "Mul_189/x" input: "bert/encoder/layer_1/output/dense/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_190/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_190" op: "Mul" input: "Mul_190/x" input: "clip_by_global_norm/clip_by_global_norm/_34" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_120" op: "Add" input: "Mul_189" input: "Mul_190" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_191/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_191" op: "Mul" input: "Mul_191/x" input: "bert/encoder/layer_1/output/dense/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_34" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_34" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_192/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_192" op: "Mul" input: "Mul_192/x" input: "Square_34" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_121" op: "Add" input: "Mul_191" input: "Mul_192" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_34" op: "Sqrt" input: "add_121" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_122/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_122" op: "Add" input: "Sqrt_34" input: "add_122/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_35" op: "RealDiv" input: "add_120" input: "add_122" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_193" op: "Mul" input: "add_2" input: "truediv_35" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_35" op: "Sub" input: "bert/encoder/layer_1/output/dense/bias/read" input: "mul_193" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_308" op: "Assign" input: "bert/encoder/layer_1/output/dense/bias" input: "sub_35" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_309" op: "Assign" input: "bert/encoder/layer_1/output/dense/bias/adam_m" input: "add_120" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_310" op: "Assign" input: "bert/encoder/layer_1/output/dense/bias/adam_v" input: "add_121" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/output/LayerNorm/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/output/LayerNorm/beta/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/output/LayerNorm/beta/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_1/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_1/output/LayerNorm/beta/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/output/LayerNorm/beta/adam_m/read" op: "Identity" input: "bert/encoder/layer_1/output/LayerNorm/beta/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/output/LayerNorm/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/output/LayerNorm/beta/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/output/LayerNorm/beta/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_1/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_1/output/LayerNorm/beta/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/output/LayerNorm/beta/adam_v/read" op: "Identity" input: "bert/encoder/layer_1/output/LayerNorm/beta/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_194/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_194" op: "Mul" input: "Mul_194/x" input: "bert/encoder/layer_1/output/LayerNorm/beta/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_195/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_195" op: "Mul" input: "Mul_195/x" input: "clip_by_global_norm/clip_by_global_norm/_35" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_123" op: "Add" input: "Mul_194" input: "Mul_195" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_196/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_196" op: "Mul" input: "Mul_196/x" input: "bert/encoder/layer_1/output/LayerNorm/beta/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_35" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_35" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_197/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_197" op: "Mul" input: "Mul_197/x" input: "Square_35" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_124" op: "Add" input: "Mul_196" input: "Mul_197" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_35" op: "Sqrt" input: "add_124" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_125/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_125" op: "Add" input: "Sqrt_35" input: "add_125/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_36" op: "RealDiv" input: "add_123" input: "add_125" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_198" op: "Mul" input: "add_2" input: "truediv_36" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_36" op: "Sub" input: "bert/encoder/layer_1/output/LayerNorm/beta/read" input: "mul_198" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_311" op: "Assign" input: "bert/encoder/layer_1/output/LayerNorm/beta" input: "sub_36" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_312" op: "Assign" input: "bert/encoder/layer_1/output/LayerNorm/beta/adam_m" input: "add_123" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_313" op: "Assign" input: "bert/encoder/layer_1/output/LayerNorm/beta/adam_v" input: "add_124" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/output/LayerNorm/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/output/LayerNorm/gamma/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/output/LayerNorm/gamma/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_1/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_1/output/LayerNorm/gamma/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/output/LayerNorm/gamma/adam_m/read" op: "Identity" input: "bert/encoder/layer_1/output/LayerNorm/gamma/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/output/LayerNorm/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/output/LayerNorm/gamma/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_1/output/LayerNorm/gamma/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_1/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_1/output/LayerNorm/gamma/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_1/output/LayerNorm/gamma/adam_v/read" op: "Identity" input: "bert/encoder/layer_1/output/LayerNorm/gamma/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_199/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_199" op: "Mul" input: "Mul_199/x" input: "bert/encoder/layer_1/output/LayerNorm/gamma/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_200/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_200" op: "Mul" input: "Mul_200/x" input: "clip_by_global_norm/clip_by_global_norm/_36" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_126" op: "Add" input: "Mul_199" input: "Mul_200" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_201/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_201" op: "Mul" input: "Mul_201/x" input: "bert/encoder/layer_1/output/LayerNorm/gamma/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_36" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_36" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_202/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_202" op: "Mul" input: "Mul_202/x" input: "Square_36" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_127" op: "Add" input: "Mul_201" input: "Mul_202" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_36" op: "Sqrt" input: "add_127" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_128/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_128" op: "Add" input: "Sqrt_36" input: "add_128/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_37" op: "RealDiv" input: "add_126" input: "add_128" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_203" op: "Mul" input: "add_2" input: "truediv_37" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_37" op: "Sub" input: "bert/encoder/layer_1/output/LayerNorm/gamma/read" input: "mul_203" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_314" op: "Assign" input: "bert/encoder/layer_1/output/LayerNorm/gamma" input: "sub_37" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_315" op: "Assign" input: "bert/encoder/layer_1/output/LayerNorm/gamma/adam_m" input: "add_126" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_316" op: "Assign" input: "bert/encoder/layer_1/output/LayerNorm/gamma/adam_v" input: "add_127" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_m" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_v" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_204/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_204" op: "Mul" input: "Mul_204/x" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_205/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_205" op: "Mul" input: "Mul_205/x" input: "clip_by_global_norm/clip_by_global_norm/_37" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_129" op: "Add" input: "Mul_204" input: "Mul_205" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_206/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_206" op: "Mul" input: "Mul_206/x" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_37" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_37" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_207/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_207" op: "Mul" input: "Mul_207/x" input: "Square_37" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_130" op: "Add" input: "Mul_206" input: "Mul_207" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_37" op: "Sqrt" input: "add_130" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_131/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_131" op: "Add" input: "Sqrt_37" input: "add_131/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_38" op: "RealDiv" input: "add_129" input: "add_131" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_208/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_208" op: "Mul" input: "mul_208/x" input: "bert/encoder/layer_2/attention/self/query/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_132" op: "Add" input: "truediv_38" input: "mul_208" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_209" op: "Mul" input: "add_2" input: "add_132" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_38" op: "Sub" input: "bert/encoder/layer_2/attention/self/query/kernel/read" input: "mul_209" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_317" op: "Assign" input: "bert/encoder/layer_2/attention/self/query/kernel" input: "sub_38" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_318" op: "Assign" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_m" input: "add_129" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_319" op: "Assign" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_v" input: "add_130" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/attention/self/query/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/self/query/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/attention/self/query/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_2/attention/self/query/bias/adam_m" input: "bert/encoder/layer_2/attention/self/query/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/attention/self/query/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_2/attention/self/query/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/self/query/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/self/query/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/attention/self/query/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_2/attention/self/query/bias/adam_v" input: "bert/encoder/layer_2/attention/self/query/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/attention/self/query/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_2/attention/self/query/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_210/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_210" op: "Mul" input: "Mul_210/x" input: "bert/encoder/layer_2/attention/self/query/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_211/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_211" op: "Mul" input: "Mul_211/x" input: "clip_by_global_norm/clip_by_global_norm/_38" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_133" op: "Add" input: "Mul_210" input: "Mul_211" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_212/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_212" op: "Mul" input: "Mul_212/x" input: "bert/encoder/layer_2/attention/self/query/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_38" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_38" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_213/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_213" op: "Mul" input: "Mul_213/x" input: "Square_38" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_134" op: "Add" input: "Mul_212" input: "Mul_213" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_38" op: "Sqrt" input: "add_134" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_135/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_135" op: "Add" input: "Sqrt_38" input: "add_135/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_39" op: "RealDiv" input: "add_133" input: "add_135" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_214" op: "Mul" input: "add_2" input: "truediv_39" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_39" op: "Sub" input: "bert/encoder/layer_2/attention/self/query/bias/read" input: "mul_214" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_320" op: "Assign" input: "bert/encoder/layer_2/attention/self/query/bias" input: "sub_39" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_321" op: "Assign" input: "bert/encoder/layer_2/attention/self/query/bias/adam_m" input: "add_133" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_322" op: "Assign" input: "bert/encoder/layer_2/attention/self/query/bias/adam_v" input: "add_134" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_m" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_v" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_215/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_215" op: "Mul" input: "Mul_215/x" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_216/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_216" op: "Mul" input: "Mul_216/x" input: "clip_by_global_norm/clip_by_global_norm/_39" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_136" op: "Add" input: "Mul_215" input: "Mul_216" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_217/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_217" op: "Mul" input: "Mul_217/x" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_39" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_39" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_218/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_218" op: "Mul" input: "Mul_218/x" input: "Square_39" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_137" op: "Add" input: "Mul_217" input: "Mul_218" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_39" op: "Sqrt" input: "add_137" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_138/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_138" op: "Add" input: "Sqrt_39" input: "add_138/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_40" op: "RealDiv" input: "add_136" input: "add_138" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_219/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_219" op: "Mul" input: "mul_219/x" input: "bert/encoder/layer_2/attention/self/key/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_139" op: "Add" input: "truediv_40" input: "mul_219" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_220" op: "Mul" input: "add_2" input: "add_139" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_40" op: "Sub" input: "bert/encoder/layer_2/attention/self/key/kernel/read" input: "mul_220" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_323" op: "Assign" input: "bert/encoder/layer_2/attention/self/key/kernel" input: "sub_40" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_324" op: "Assign" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_m" input: "add_136" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_325" op: "Assign" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_v" input: "add_137" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/attention/self/key/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/self/key/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/attention/self/key/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_2/attention/self/key/bias/adam_m" input: "bert/encoder/layer_2/attention/self/key/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/attention/self/key/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_2/attention/self/key/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/self/key/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/self/key/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/attention/self/key/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_2/attention/self/key/bias/adam_v" input: "bert/encoder/layer_2/attention/self/key/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/attention/self/key/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_2/attention/self/key/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_221/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_221" op: "Mul" input: "Mul_221/x" input: "bert/encoder/layer_2/attention/self/key/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_222/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_222" op: "Mul" input: "Mul_222/x" input: "clip_by_global_norm/clip_by_global_norm/_40" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_140" op: "Add" input: "Mul_221" input: "Mul_222" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_223/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_223" op: "Mul" input: "Mul_223/x" input: "bert/encoder/layer_2/attention/self/key/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_40" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_40" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_224/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_224" op: "Mul" input: "Mul_224/x" input: "Square_40" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_141" op: "Add" input: "Mul_223" input: "Mul_224" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_40" op: "Sqrt" input: "add_141" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_142/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_142" op: "Add" input: "Sqrt_40" input: "add_142/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_41" op: "RealDiv" input: "add_140" input: "add_142" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_225" op: "Mul" input: "add_2" input: "truediv_41" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_41" op: "Sub" input: "bert/encoder/layer_2/attention/self/key/bias/read" input: "mul_225" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_326" op: "Assign" input: "bert/encoder/layer_2/attention/self/key/bias" input: "sub_41" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_327" op: "Assign" input: "bert/encoder/layer_2/attention/self/key/bias/adam_m" input: "add_140" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_328" op: "Assign" input: "bert/encoder/layer_2/attention/self/key/bias/adam_v" input: "add_141" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_m" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_v" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_226/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_226" op: "Mul" input: "Mul_226/x" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_227/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_227" op: "Mul" input: "Mul_227/x" input: "clip_by_global_norm/clip_by_global_norm/_41" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_143" op: "Add" input: "Mul_226" input: "Mul_227" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_228/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_228" op: "Mul" input: "Mul_228/x" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_41" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_41" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_229/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_229" op: "Mul" input: "Mul_229/x" input: "Square_41" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_144" op: "Add" input: "Mul_228" input: "Mul_229" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_41" op: "Sqrt" input: "add_144" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_145/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_145" op: "Add" input: "Sqrt_41" input: "add_145/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_42" op: "RealDiv" input: "add_143" input: "add_145" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_230/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_230" op: "Mul" input: "mul_230/x" input: "bert/encoder/layer_2/attention/self/value/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_146" op: "Add" input: "truediv_42" input: "mul_230" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_231" op: "Mul" input: "add_2" input: "add_146" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_42" op: "Sub" input: "bert/encoder/layer_2/attention/self/value/kernel/read" input: "mul_231" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_329" op: "Assign" input: "bert/encoder/layer_2/attention/self/value/kernel" input: "sub_42" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_330" op: "Assign" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_m" input: "add_143" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_331" op: "Assign" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_v" input: "add_144" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/attention/self/value/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/self/value/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/attention/self/value/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_2/attention/self/value/bias/adam_m" input: "bert/encoder/layer_2/attention/self/value/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/attention/self/value/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_2/attention/self/value/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/self/value/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/self/value/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/attention/self/value/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_2/attention/self/value/bias/adam_v" input: "bert/encoder/layer_2/attention/self/value/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/attention/self/value/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_2/attention/self/value/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_232/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_232" op: "Mul" input: "Mul_232/x" input: "bert/encoder/layer_2/attention/self/value/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_233/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_233" op: "Mul" input: "Mul_233/x" input: "clip_by_global_norm/clip_by_global_norm/_42" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_147" op: "Add" input: "Mul_232" input: "Mul_233" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_234/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_234" op: "Mul" input: "Mul_234/x" input: "bert/encoder/layer_2/attention/self/value/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_42" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_42" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_235/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_235" op: "Mul" input: "Mul_235/x" input: "Square_42" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_148" op: "Add" input: "Mul_234" input: "Mul_235" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_42" op: "Sqrt" input: "add_148" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_149/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_149" op: "Add" input: "Sqrt_42" input: "add_149/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_43" op: "RealDiv" input: "add_147" input: "add_149" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_236" op: "Mul" input: "add_2" input: "truediv_43" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_43" op: "Sub" input: "bert/encoder/layer_2/attention/self/value/bias/read" input: "mul_236" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_332" op: "Assign" input: "bert/encoder/layer_2/attention/self/value/bias" input: "sub_43" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_333" op: "Assign" input: "bert/encoder/layer_2/attention/self/value/bias/adam_m" input: "add_147" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_334" op: "Assign" input: "bert/encoder/layer_2/attention/self/value/bias/adam_v" input: "add_148" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_237/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_237" op: "Mul" input: "Mul_237/x" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_238/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_238" op: "Mul" input: "Mul_238/x" input: "clip_by_global_norm/clip_by_global_norm/_43" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_150" op: "Add" input: "Mul_237" input: "Mul_238" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_239/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_239" op: "Mul" input: "Mul_239/x" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_43" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_43" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_240/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_240" op: "Mul" input: "Mul_240/x" input: "Square_43" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_151" op: "Add" input: "Mul_239" input: "Mul_240" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_43" op: "Sqrt" input: "add_151" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_152/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_152" op: "Add" input: "Sqrt_43" input: "add_152/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_44" op: "RealDiv" input: "add_150" input: "add_152" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_241/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_241" op: "Mul" input: "mul_241/x" input: "bert/encoder/layer_2/attention/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_153" op: "Add" input: "truediv_44" input: "mul_241" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_242" op: "Mul" input: "add_2" input: "add_153" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_44" op: "Sub" input: "bert/encoder/layer_2/attention/output/dense/kernel/read" input: "mul_242" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_335" op: "Assign" input: "bert/encoder/layer_2/attention/output/dense/kernel" input: "sub_44" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_336" op: "Assign" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m" input: "add_150" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_337" op: "Assign" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v" input: "add_151" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/attention/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/output/dense/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/attention/output/dense/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_2/attention/output/dense/bias/adam_m" input: "bert/encoder/layer_2/attention/output/dense/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/attention/output/dense/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_2/attention/output/dense/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/output/dense/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/attention/output/dense/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_2/attention/output/dense/bias/adam_v" input: "bert/encoder/layer_2/attention/output/dense/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/attention/output/dense/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_2/attention/output/dense/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_243/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_243" op: "Mul" input: "Mul_243/x" input: "bert/encoder/layer_2/attention/output/dense/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_244/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_244" op: "Mul" input: "Mul_244/x" input: "clip_by_global_norm/clip_by_global_norm/_44" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_154" op: "Add" input: "Mul_243" input: "Mul_244" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_245/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_245" op: "Mul" input: "Mul_245/x" input: "bert/encoder/layer_2/attention/output/dense/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_44" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_44" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_246/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_246" op: "Mul" input: "Mul_246/x" input: "Square_44" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_155" op: "Add" input: "Mul_245" input: "Mul_246" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_44" op: "Sqrt" input: "add_155" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_156/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_156" op: "Add" input: "Sqrt_44" input: "add_156/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_45" op: "RealDiv" input: "add_154" input: "add_156" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_247" op: "Mul" input: "add_2" input: "truediv_45" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_45" op: "Sub" input: "bert/encoder/layer_2/attention/output/dense/bias/read" input: "mul_247" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_338" op: "Assign" input: "bert/encoder/layer_2/attention/output/dense/bias" input: "sub_45" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_339" op: "Assign" input: "bert/encoder/layer_2/attention/output/dense/bias/adam_m" input: "add_154" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_340" op: "Assign" input: "bert/encoder/layer_2/attention/output/dense/bias/adam_v" input: "add_155" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_m/read" op: "Identity" input: "bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_v/read" op: "Identity" input: "bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_248/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_248" op: "Mul" input: "Mul_248/x" input: "bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_249/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_249" op: "Mul" input: "Mul_249/x" input: "clip_by_global_norm/clip_by_global_norm/_45" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_157" op: "Add" input: "Mul_248" input: "Mul_249" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_250/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_250" op: "Mul" input: "Mul_250/x" input: "bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_45" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_45" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_251/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_251" op: "Mul" input: "Mul_251/x" input: "Square_45" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_158" op: "Add" input: "Mul_250" input: "Mul_251" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_45" op: "Sqrt" input: "add_158" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_159/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_159" op: "Add" input: "Sqrt_45" input: "add_159/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_46" op: "RealDiv" input: "add_157" input: "add_159" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_252" op: "Mul" input: "add_2" input: "truediv_46" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_46" op: "Sub" input: "bert/encoder/layer_2/attention/output/LayerNorm/beta/read" input: "mul_252" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_341" op: "Assign" input: "bert/encoder/layer_2/attention/output/LayerNorm/beta" input: "sub_46" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_342" op: "Assign" input: "bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_m" input: "add_157" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_343" op: "Assign" input: "bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_v" input: "add_158" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m/read" op: "Identity" input: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v/read" op: "Identity" input: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_253/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_253" op: "Mul" input: "Mul_253/x" input: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_254/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_254" op: "Mul" input: "Mul_254/x" input: "clip_by_global_norm/clip_by_global_norm/_46" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_160" op: "Add" input: "Mul_253" input: "Mul_254" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_255/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_255" op: "Mul" input: "Mul_255/x" input: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_46" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_46" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_256/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_256" op: "Mul" input: "Mul_256/x" input: "Square_46" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_161" op: "Add" input: "Mul_255" input: "Mul_256" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_46" op: "Sqrt" input: "add_161" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_162/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_162" op: "Add" input: "Sqrt_46" input: "add_162/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_47" op: "RealDiv" input: "add_160" input: "add_162" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_257" op: "Mul" input: "add_2" input: "truediv_47" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_47" op: "Sub" input: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/read" input: "mul_257" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_344" op: "Assign" input: "bert/encoder/layer_2/attention/output/LayerNorm/gamma" input: "sub_47" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_345" op: "Assign" input: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m" input: "add_160" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_346" op: "Assign" input: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v" input: "add_161" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_258/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_258" op: "Mul" input: "Mul_258/x" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_259/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_259" op: "Mul" input: "Mul_259/x" input: "clip_by_global_norm/clip_by_global_norm/_47" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_163" op: "Add" input: "Mul_258" input: "Mul_259" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_260/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_260" op: "Mul" input: "Mul_260/x" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Square_47" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_47" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_261/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_261" op: "Mul" input: "Mul_261/x" input: "Square_47" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_164" op: "Add" input: "Mul_260" input: "Mul_261" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Sqrt_47" op: "Sqrt" input: "add_164" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_165/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_165" op: "Add" input: "Sqrt_47" input: "add_165/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "truediv_48" op: "RealDiv" input: "add_163" input: "add_165" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "mul_262/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_262" op: "Mul" input: "mul_262/x" input: "bert/encoder/layer_2/intermediate/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_166" op: "Add" input: "truediv_48" input: "mul_262" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "mul_263" op: "Mul" input: "add_2" input: "add_166" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "sub_48" op: "Sub" input: "bert/encoder/layer_2/intermediate/dense/kernel/read" input: "mul_263" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Assign_347" op: "Assign" input: "bert/encoder/layer_2/intermediate/dense/kernel" input: "sub_48" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_348" op: "Assign" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m" input: "add_163" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_349" op: "Assign" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v" input: "add_164" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_2/intermediate/dense/bias/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/intermediate/dense/bias/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_2/intermediate/dense/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/intermediate/dense/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_m" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/intermediate/dense/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_2/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_2/intermediate/dense/bias/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/intermediate/dense/bias/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_2/intermediate/dense/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/intermediate/dense/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_v" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/intermediate/dense/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_264/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_264" op: "Mul" input: "Mul_264/x" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_265/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_265" op: "Mul" input: "Mul_265/x" input: "clip_by_global_norm/clip_by_global_norm/_48" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_167" op: "Add" input: "Mul_264" input: "Mul_265" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_266/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_266" op: "Mul" input: "Mul_266/x" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Square_48" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_48" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_267/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_267" op: "Mul" input: "Mul_267/x" input: "Square_48" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_168" op: "Add" input: "Mul_266" input: "Mul_267" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Sqrt_48" op: "Sqrt" input: "add_168" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_169/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_169" op: "Add" input: "Sqrt_48" input: "add_169/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "truediv_49" op: "RealDiv" input: "add_167" input: "add_169" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "mul_268" op: "Mul" input: "add_2" input: "truediv_49" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "sub_49" op: "Sub" input: "bert/encoder/layer_2/intermediate/dense/bias/read" input: "mul_268" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Assign_350" op: "Assign" input: "bert/encoder/layer_2/intermediate/dense/bias" input: "sub_49" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_351" op: "Assign" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_m" input: "add_167" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_352" op: "Assign" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_v" input: "add_168" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_2/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_2/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_2/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_2/output/dense/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/output/dense/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_2/output/dense/kernel/adam_m" input: "bert/encoder/layer_2/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/output/dense/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_2/output/dense/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_2/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_2/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_2/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_2/output/dense/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/output/dense/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_2/output/dense/kernel/adam_v" input: "bert/encoder/layer_2/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/output/dense/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_2/output/dense/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_269/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_269" op: "Mul" input: "Mul_269/x" input: "bert/encoder/layer_2/output/dense/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_270/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_270" op: "Mul" input: "Mul_270/x" input: "clip_by_global_norm/clip_by_global_norm/_49" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_170" op: "Add" input: "Mul_269" input: "Mul_270" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_271/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_271" op: "Mul" input: "Mul_271/x" input: "bert/encoder/layer_2/output/dense/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Square_49" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_49" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_272/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_272" op: "Mul" input: "Mul_272/x" input: "Square_49" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_171" op: "Add" input: "Mul_271" input: "Mul_272" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Sqrt_49" op: "Sqrt" input: "add_171" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_172/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_172" op: "Add" input: "Sqrt_49" input: "add_172/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "truediv_50" op: "RealDiv" input: "add_170" input: "add_172" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "mul_273/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_273" op: "Mul" input: "mul_273/x" input: "bert/encoder/layer_2/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_173" op: "Add" input: "truediv_50" input: "mul_273" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "mul_274" op: "Mul" input: "add_2" input: "add_173" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "sub_50" op: "Sub" input: "bert/encoder/layer_2/output/dense/kernel/read" input: "mul_274" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Assign_353" op: "Assign" input: "bert/encoder/layer_2/output/dense/kernel" input: "sub_50" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_354" op: "Assign" input: "bert/encoder/layer_2/output/dense/kernel/adam_m" input: "add_170" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_355" op: "Assign" input: "bert/encoder/layer_2/output/dense/kernel/adam_v" input: "add_171" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/output/dense/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/output/dense/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_2/output/dense/bias/adam_m" input: "bert/encoder/layer_2/output/dense/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/output/dense/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_2/output/dense/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/output/dense/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/output/dense/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_2/output/dense/bias/adam_v" input: "bert/encoder/layer_2/output/dense/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/output/dense/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_2/output/dense/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_275/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_275" op: "Mul" input: "Mul_275/x" input: "bert/encoder/layer_2/output/dense/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_276/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_276" op: "Mul" input: "Mul_276/x" input: "clip_by_global_norm/clip_by_global_norm/_50" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_174" op: "Add" input: "Mul_275" input: "Mul_276" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_277/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_277" op: "Mul" input: "Mul_277/x" input: "bert/encoder/layer_2/output/dense/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_50" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_50" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_278/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_278" op: "Mul" input: "Mul_278/x" input: "Square_50" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_175" op: "Add" input: "Mul_277" input: "Mul_278" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_50" op: "Sqrt" input: "add_175" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_176/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_176" op: "Add" input: "Sqrt_50" input: "add_176/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_51" op: "RealDiv" input: "add_174" input: "add_176" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_279" op: "Mul" input: "add_2" input: "truediv_51" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_51" op: "Sub" input: "bert/encoder/layer_2/output/dense/bias/read" input: "mul_279" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_356" op: "Assign" input: "bert/encoder/layer_2/output/dense/bias" input: "sub_51" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_357" op: "Assign" input: "bert/encoder/layer_2/output/dense/bias/adam_m" input: "add_174" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_358" op: "Assign" input: "bert/encoder/layer_2/output/dense/bias/adam_v" input: "add_175" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/output/LayerNorm/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/output/LayerNorm/beta/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/output/LayerNorm/beta/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_2/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_2/output/LayerNorm/beta/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/output/LayerNorm/beta/adam_m/read" op: "Identity" input: "bert/encoder/layer_2/output/LayerNorm/beta/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/output/LayerNorm/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/output/LayerNorm/beta/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/output/LayerNorm/beta/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_2/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_2/output/LayerNorm/beta/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/output/LayerNorm/beta/adam_v/read" op: "Identity" input: "bert/encoder/layer_2/output/LayerNorm/beta/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_280/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_280" op: "Mul" input: "Mul_280/x" input: "bert/encoder/layer_2/output/LayerNorm/beta/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_281/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_281" op: "Mul" input: "Mul_281/x" input: "clip_by_global_norm/clip_by_global_norm/_51" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_177" op: "Add" input: "Mul_280" input: "Mul_281" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_282/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_282" op: "Mul" input: "Mul_282/x" input: "bert/encoder/layer_2/output/LayerNorm/beta/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_51" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_51" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_283/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_283" op: "Mul" input: "Mul_283/x" input: "Square_51" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_178" op: "Add" input: "Mul_282" input: "Mul_283" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_51" op: "Sqrt" input: "add_178" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_179/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_179" op: "Add" input: "Sqrt_51" input: "add_179/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_52" op: "RealDiv" input: "add_177" input: "add_179" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_284" op: "Mul" input: "add_2" input: "truediv_52" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_52" op: "Sub" input: "bert/encoder/layer_2/output/LayerNorm/beta/read" input: "mul_284" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_359" op: "Assign" input: "bert/encoder/layer_2/output/LayerNorm/beta" input: "sub_52" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_360" op: "Assign" input: "bert/encoder/layer_2/output/LayerNorm/beta/adam_m" input: "add_177" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_361" op: "Assign" input: "bert/encoder/layer_2/output/LayerNorm/beta/adam_v" input: "add_178" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/output/LayerNorm/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/output/LayerNorm/gamma/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/output/LayerNorm/gamma/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_2/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_2/output/LayerNorm/gamma/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/output/LayerNorm/gamma/adam_m/read" op: "Identity" input: "bert/encoder/layer_2/output/LayerNorm/gamma/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/output/LayerNorm/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/output/LayerNorm/gamma/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_2/output/LayerNorm/gamma/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_2/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_2/output/LayerNorm/gamma/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_2/output/LayerNorm/gamma/adam_v/read" op: "Identity" input: "bert/encoder/layer_2/output/LayerNorm/gamma/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_285/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_285" op: "Mul" input: "Mul_285/x" input: "bert/encoder/layer_2/output/LayerNorm/gamma/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_286/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_286" op: "Mul" input: "Mul_286/x" input: "clip_by_global_norm/clip_by_global_norm/_52" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_180" op: "Add" input: "Mul_285" input: "Mul_286" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_287/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_287" op: "Mul" input: "Mul_287/x" input: "bert/encoder/layer_2/output/LayerNorm/gamma/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_52" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_52" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_288/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_288" op: "Mul" input: "Mul_288/x" input: "Square_52" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_181" op: "Add" input: "Mul_287" input: "Mul_288" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_52" op: "Sqrt" input: "add_181" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_182/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_182" op: "Add" input: "Sqrt_52" input: "add_182/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_53" op: "RealDiv" input: "add_180" input: "add_182" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_289" op: "Mul" input: "add_2" input: "truediv_53" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_53" op: "Sub" input: "bert/encoder/layer_2/output/LayerNorm/gamma/read" input: "mul_289" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_362" op: "Assign" input: "bert/encoder/layer_2/output/LayerNorm/gamma" input: "sub_53" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_363" op: "Assign" input: "bert/encoder/layer_2/output/LayerNorm/gamma/adam_m" input: "add_180" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_364" op: "Assign" input: "bert/encoder/layer_2/output/LayerNorm/gamma/adam_v" input: "add_181" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_m" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_v" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_290/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_290" op: "Mul" input: "Mul_290/x" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_291/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_291" op: "Mul" input: "Mul_291/x" input: "clip_by_global_norm/clip_by_global_norm/_53" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_183" op: "Add" input: "Mul_290" input: "Mul_291" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_292/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_292" op: "Mul" input: "Mul_292/x" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_53" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_53" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_293/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_293" op: "Mul" input: "Mul_293/x" input: "Square_53" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_184" op: "Add" input: "Mul_292" input: "Mul_293" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_53" op: "Sqrt" input: "add_184" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_185/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_185" op: "Add" input: "Sqrt_53" input: "add_185/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_54" op: "RealDiv" input: "add_183" input: "add_185" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_294/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_294" op: "Mul" input: "mul_294/x" input: "bert/encoder/layer_3/attention/self/query/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_186" op: "Add" input: "truediv_54" input: "mul_294" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_295" op: "Mul" input: "add_2" input: "add_186" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_54" op: "Sub" input: "bert/encoder/layer_3/attention/self/query/kernel/read" input: "mul_295" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_365" op: "Assign" input: "bert/encoder/layer_3/attention/self/query/kernel" input: "sub_54" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_366" op: "Assign" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_m" input: "add_183" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_367" op: "Assign" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_v" input: "add_184" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/attention/self/query/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/self/query/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/attention/self/query/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_3/attention/self/query/bias/adam_m" input: "bert/encoder/layer_3/attention/self/query/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/attention/self/query/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_3/attention/self/query/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/self/query/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/self/query/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/attention/self/query/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_3/attention/self/query/bias/adam_v" input: "bert/encoder/layer_3/attention/self/query/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/attention/self/query/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_3/attention/self/query/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_296/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_296" op: "Mul" input: "Mul_296/x" input: "bert/encoder/layer_3/attention/self/query/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_297/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_297" op: "Mul" input: "Mul_297/x" input: "clip_by_global_norm/clip_by_global_norm/_54" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_187" op: "Add" input: "Mul_296" input: "Mul_297" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_298/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_298" op: "Mul" input: "Mul_298/x" input: "bert/encoder/layer_3/attention/self/query/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_54" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_54" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_299/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_299" op: "Mul" input: "Mul_299/x" input: "Square_54" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_188" op: "Add" input: "Mul_298" input: "Mul_299" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_54" op: "Sqrt" input: "add_188" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_189/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_189" op: "Add" input: "Sqrt_54" input: "add_189/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_55" op: "RealDiv" input: "add_187" input: "add_189" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_300" op: "Mul" input: "add_2" input: "truediv_55" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_55" op: "Sub" input: "bert/encoder/layer_3/attention/self/query/bias/read" input: "mul_300" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_368" op: "Assign" input: "bert/encoder/layer_3/attention/self/query/bias" input: "sub_55" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_369" op: "Assign" input: "bert/encoder/layer_3/attention/self/query/bias/adam_m" input: "add_187" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_370" op: "Assign" input: "bert/encoder/layer_3/attention/self/query/bias/adam_v" input: "add_188" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_m" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_v" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_301/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_301" op: "Mul" input: "Mul_301/x" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_302/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_302" op: "Mul" input: "Mul_302/x" input: "clip_by_global_norm/clip_by_global_norm/_55" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_190" op: "Add" input: "Mul_301" input: "Mul_302" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_303/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_303" op: "Mul" input: "Mul_303/x" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_55" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_55" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_304/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_304" op: "Mul" input: "Mul_304/x" input: "Square_55" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_191" op: "Add" input: "Mul_303" input: "Mul_304" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_55" op: "Sqrt" input: "add_191" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_192/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_192" op: "Add" input: "Sqrt_55" input: "add_192/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_56" op: "RealDiv" input: "add_190" input: "add_192" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_305/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_305" op: "Mul" input: "mul_305/x" input: "bert/encoder/layer_3/attention/self/key/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_193" op: "Add" input: "truediv_56" input: "mul_305" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_306" op: "Mul" input: "add_2" input: "add_193" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_56" op: "Sub" input: "bert/encoder/layer_3/attention/self/key/kernel/read" input: "mul_306" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_371" op: "Assign" input: "bert/encoder/layer_3/attention/self/key/kernel" input: "sub_56" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_372" op: "Assign" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_m" input: "add_190" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_373" op: "Assign" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_v" input: "add_191" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/attention/self/key/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/self/key/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/attention/self/key/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_3/attention/self/key/bias/adam_m" input: "bert/encoder/layer_3/attention/self/key/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/attention/self/key/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_3/attention/self/key/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/self/key/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/self/key/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/attention/self/key/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_3/attention/self/key/bias/adam_v" input: "bert/encoder/layer_3/attention/self/key/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/attention/self/key/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_3/attention/self/key/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_307/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_307" op: "Mul" input: "Mul_307/x" input: "bert/encoder/layer_3/attention/self/key/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_308/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_308" op: "Mul" input: "Mul_308/x" input: "clip_by_global_norm/clip_by_global_norm/_56" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_194" op: "Add" input: "Mul_307" input: "Mul_308" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_309/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_309" op: "Mul" input: "Mul_309/x" input: "bert/encoder/layer_3/attention/self/key/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_56" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_56" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_310/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_310" op: "Mul" input: "Mul_310/x" input: "Square_56" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_195" op: "Add" input: "Mul_309" input: "Mul_310" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_56" op: "Sqrt" input: "add_195" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_196/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_196" op: "Add" input: "Sqrt_56" input: "add_196/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_57" op: "RealDiv" input: "add_194" input: "add_196" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_311" op: "Mul" input: "add_2" input: "truediv_57" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_57" op: "Sub" input: "bert/encoder/layer_3/attention/self/key/bias/read" input: "mul_311" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_374" op: "Assign" input: "bert/encoder/layer_3/attention/self/key/bias" input: "sub_57" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_375" op: "Assign" input: "bert/encoder/layer_3/attention/self/key/bias/adam_m" input: "add_194" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_376" op: "Assign" input: "bert/encoder/layer_3/attention/self/key/bias/adam_v" input: "add_195" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_m" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_v" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_312/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_312" op: "Mul" input: "Mul_312/x" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_313/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_313" op: "Mul" input: "Mul_313/x" input: "clip_by_global_norm/clip_by_global_norm/_57" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_197" op: "Add" input: "Mul_312" input: "Mul_313" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_314/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_314" op: "Mul" input: "Mul_314/x" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_57" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_57" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_315/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_315" op: "Mul" input: "Mul_315/x" input: "Square_57" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_198" op: "Add" input: "Mul_314" input: "Mul_315" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_57" op: "Sqrt" input: "add_198" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_199/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_199" op: "Add" input: "Sqrt_57" input: "add_199/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_58" op: "RealDiv" input: "add_197" input: "add_199" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_316/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_316" op: "Mul" input: "mul_316/x" input: "bert/encoder/layer_3/attention/self/value/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_200" op: "Add" input: "truediv_58" input: "mul_316" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_317" op: "Mul" input: "add_2" input: "add_200" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_58" op: "Sub" input: "bert/encoder/layer_3/attention/self/value/kernel/read" input: "mul_317" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_377" op: "Assign" input: "bert/encoder/layer_3/attention/self/value/kernel" input: "sub_58" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_378" op: "Assign" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_m" input: "add_197" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_379" op: "Assign" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_v" input: "add_198" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/attention/self/value/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/self/value/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/attention/self/value/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_3/attention/self/value/bias/adam_m" input: "bert/encoder/layer_3/attention/self/value/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/attention/self/value/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_3/attention/self/value/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/self/value/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/self/value/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/attention/self/value/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_3/attention/self/value/bias/adam_v" input: "bert/encoder/layer_3/attention/self/value/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/attention/self/value/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_3/attention/self/value/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_318/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_318" op: "Mul" input: "Mul_318/x" input: "bert/encoder/layer_3/attention/self/value/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_319/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_319" op: "Mul" input: "Mul_319/x" input: "clip_by_global_norm/clip_by_global_norm/_58" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_201" op: "Add" input: "Mul_318" input: "Mul_319" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_320/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_320" op: "Mul" input: "Mul_320/x" input: "bert/encoder/layer_3/attention/self/value/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_58" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_58" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_321/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_321" op: "Mul" input: "Mul_321/x" input: "Square_58" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_202" op: "Add" input: "Mul_320" input: "Mul_321" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_58" op: "Sqrt" input: "add_202" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_203/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_203" op: "Add" input: "Sqrt_58" input: "add_203/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_59" op: "RealDiv" input: "add_201" input: "add_203" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_322" op: "Mul" input: "add_2" input: "truediv_59" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_59" op: "Sub" input: "bert/encoder/layer_3/attention/self/value/bias/read" input: "mul_322" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_380" op: "Assign" input: "bert/encoder/layer_3/attention/self/value/bias" input: "sub_59" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_381" op: "Assign" input: "bert/encoder/layer_3/attention/self/value/bias/adam_m" input: "add_201" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_382" op: "Assign" input: "bert/encoder/layer_3/attention/self/value/bias/adam_v" input: "add_202" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_323/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_323" op: "Mul" input: "Mul_323/x" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_324/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_324" op: "Mul" input: "Mul_324/x" input: "clip_by_global_norm/clip_by_global_norm/_59" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_204" op: "Add" input: "Mul_323" input: "Mul_324" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_325/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_325" op: "Mul" input: "Mul_325/x" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_59" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_59" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_326/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_326" op: "Mul" input: "Mul_326/x" input: "Square_59" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_205" op: "Add" input: "Mul_325" input: "Mul_326" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_59" op: "Sqrt" input: "add_205" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_206/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_206" op: "Add" input: "Sqrt_59" input: "add_206/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_60" op: "RealDiv" input: "add_204" input: "add_206" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_327/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_327" op: "Mul" input: "mul_327/x" input: "bert/encoder/layer_3/attention/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_207" op: "Add" input: "truediv_60" input: "mul_327" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_328" op: "Mul" input: "add_2" input: "add_207" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_60" op: "Sub" input: "bert/encoder/layer_3/attention/output/dense/kernel/read" input: "mul_328" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_383" op: "Assign" input: "bert/encoder/layer_3/attention/output/dense/kernel" input: "sub_60" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_384" op: "Assign" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m" input: "add_204" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_385" op: "Assign" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v" input: "add_205" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/attention/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/output/dense/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/attention/output/dense/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_3/attention/output/dense/bias/adam_m" input: "bert/encoder/layer_3/attention/output/dense/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/attention/output/dense/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_3/attention/output/dense/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/output/dense/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/attention/output/dense/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_3/attention/output/dense/bias/adam_v" input: "bert/encoder/layer_3/attention/output/dense/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/attention/output/dense/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_3/attention/output/dense/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_329/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_329" op: "Mul" input: "Mul_329/x" input: "bert/encoder/layer_3/attention/output/dense/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_330/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_330" op: "Mul" input: "Mul_330/x" input: "clip_by_global_norm/clip_by_global_norm/_60" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_208" op: "Add" input: "Mul_329" input: "Mul_330" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_331/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_331" op: "Mul" input: "Mul_331/x" input: "bert/encoder/layer_3/attention/output/dense/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_60" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_60" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_332/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_332" op: "Mul" input: "Mul_332/x" input: "Square_60" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_209" op: "Add" input: "Mul_331" input: "Mul_332" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_60" op: "Sqrt" input: "add_209" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_210/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_210" op: "Add" input: "Sqrt_60" input: "add_210/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_61" op: "RealDiv" input: "add_208" input: "add_210" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_333" op: "Mul" input: "add_2" input: "truediv_61" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_61" op: "Sub" input: "bert/encoder/layer_3/attention/output/dense/bias/read" input: "mul_333" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_386" op: "Assign" input: "bert/encoder/layer_3/attention/output/dense/bias" input: "sub_61" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_387" op: "Assign" input: "bert/encoder/layer_3/attention/output/dense/bias/adam_m" input: "add_208" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_388" op: "Assign" input: "bert/encoder/layer_3/attention/output/dense/bias/adam_v" input: "add_209" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_m/read" op: "Identity" input: "bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_v/read" op: "Identity" input: "bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_334/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_334" op: "Mul" input: "Mul_334/x" input: "bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_335/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_335" op: "Mul" input: "Mul_335/x" input: "clip_by_global_norm/clip_by_global_norm/_61" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_211" op: "Add" input: "Mul_334" input: "Mul_335" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_336/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_336" op: "Mul" input: "Mul_336/x" input: "bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_61" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_61" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_337/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_337" op: "Mul" input: "Mul_337/x" input: "Square_61" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_212" op: "Add" input: "Mul_336" input: "Mul_337" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_61" op: "Sqrt" input: "add_212" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_213/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_213" op: "Add" input: "Sqrt_61" input: "add_213/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_62" op: "RealDiv" input: "add_211" input: "add_213" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_338" op: "Mul" input: "add_2" input: "truediv_62" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_62" op: "Sub" input: "bert/encoder/layer_3/attention/output/LayerNorm/beta/read" input: "mul_338" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_389" op: "Assign" input: "bert/encoder/layer_3/attention/output/LayerNorm/beta" input: "sub_62" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_390" op: "Assign" input: "bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_m" input: "add_211" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_391" op: "Assign" input: "bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_v" input: "add_212" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m/read" op: "Identity" input: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v/read" op: "Identity" input: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_339/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_339" op: "Mul" input: "Mul_339/x" input: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_340/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_340" op: "Mul" input: "Mul_340/x" input: "clip_by_global_norm/clip_by_global_norm/_62" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_214" op: "Add" input: "Mul_339" input: "Mul_340" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_341/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_341" op: "Mul" input: "Mul_341/x" input: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_62" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_62" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_342/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_342" op: "Mul" input: "Mul_342/x" input: "Square_62" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_215" op: "Add" input: "Mul_341" input: "Mul_342" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_62" op: "Sqrt" input: "add_215" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_216/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_216" op: "Add" input: "Sqrt_62" input: "add_216/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_63" op: "RealDiv" input: "add_214" input: "add_216" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_343" op: "Mul" input: "add_2" input: "truediv_63" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_63" op: "Sub" input: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/read" input: "mul_343" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_392" op: "Assign" input: "bert/encoder/layer_3/attention/output/LayerNorm/gamma" input: "sub_63" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_393" op: "Assign" input: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m" input: "add_214" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_394" op: "Assign" input: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v" input: "add_215" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_344/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_344" op: "Mul" input: "Mul_344/x" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_345/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_345" op: "Mul" input: "Mul_345/x" input: "clip_by_global_norm/clip_by_global_norm/_63" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_217" op: "Add" input: "Mul_344" input: "Mul_345" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_346/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_346" op: "Mul" input: "Mul_346/x" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Square_63" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_63" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_347/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_347" op: "Mul" input: "Mul_347/x" input: "Square_63" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_218" op: "Add" input: "Mul_346" input: "Mul_347" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Sqrt_63" op: "Sqrt" input: "add_218" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_219/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_219" op: "Add" input: "Sqrt_63" input: "add_219/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "truediv_64" op: "RealDiv" input: "add_217" input: "add_219" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "mul_348/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_348" op: "Mul" input: "mul_348/x" input: "bert/encoder/layer_3/intermediate/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_220" op: "Add" input: "truediv_64" input: "mul_348" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "mul_349" op: "Mul" input: "add_2" input: "add_220" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "sub_64" op: "Sub" input: "bert/encoder/layer_3/intermediate/dense/kernel/read" input: "mul_349" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Assign_395" op: "Assign" input: "bert/encoder/layer_3/intermediate/dense/kernel" input: "sub_64" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_396" op: "Assign" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m" input: "add_217" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_397" op: "Assign" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v" input: "add_218" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_3/intermediate/dense/bias/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/intermediate/dense/bias/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_3/intermediate/dense/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/intermediate/dense/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_m" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/intermediate/dense/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_3/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_3/intermediate/dense/bias/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/intermediate/dense/bias/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_3/intermediate/dense/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/intermediate/dense/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_v" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/intermediate/dense/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_350/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_350" op: "Mul" input: "Mul_350/x" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_351/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_351" op: "Mul" input: "Mul_351/x" input: "clip_by_global_norm/clip_by_global_norm/_64" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_221" op: "Add" input: "Mul_350" input: "Mul_351" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_352/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_352" op: "Mul" input: "Mul_352/x" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Square_64" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_64" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_353/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_353" op: "Mul" input: "Mul_353/x" input: "Square_64" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_222" op: "Add" input: "Mul_352" input: "Mul_353" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Sqrt_64" op: "Sqrt" input: "add_222" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_223/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_223" op: "Add" input: "Sqrt_64" input: "add_223/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "truediv_65" op: "RealDiv" input: "add_221" input: "add_223" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "mul_354" op: "Mul" input: "add_2" input: "truediv_65" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "sub_65" op: "Sub" input: "bert/encoder/layer_3/intermediate/dense/bias/read" input: "mul_354" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Assign_398" op: "Assign" input: "bert/encoder/layer_3/intermediate/dense/bias" input: "sub_65" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_399" op: "Assign" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_m" input: "add_221" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_400" op: "Assign" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_v" input: "add_222" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_3/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_3/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_3/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_3/output/dense/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/output/dense/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_3/output/dense/kernel/adam_m" input: "bert/encoder/layer_3/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/output/dense/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_3/output/dense/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_3/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_3/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_3/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_3/output/dense/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/output/dense/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_3/output/dense/kernel/adam_v" input: "bert/encoder/layer_3/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/output/dense/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_3/output/dense/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_355/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_355" op: "Mul" input: "Mul_355/x" input: "bert/encoder/layer_3/output/dense/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_356/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_356" op: "Mul" input: "Mul_356/x" input: "clip_by_global_norm/clip_by_global_norm/_65" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_224" op: "Add" input: "Mul_355" input: "Mul_356" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_357/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_357" op: "Mul" input: "Mul_357/x" input: "bert/encoder/layer_3/output/dense/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Square_65" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_65" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_358/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_358" op: "Mul" input: "Mul_358/x" input: "Square_65" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_225" op: "Add" input: "Mul_357" input: "Mul_358" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Sqrt_65" op: "Sqrt" input: "add_225" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_226/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_226" op: "Add" input: "Sqrt_65" input: "add_226/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "truediv_66" op: "RealDiv" input: "add_224" input: "add_226" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "mul_359/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_359" op: "Mul" input: "mul_359/x" input: "bert/encoder/layer_3/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_227" op: "Add" input: "truediv_66" input: "mul_359" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "mul_360" op: "Mul" input: "add_2" input: "add_227" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "sub_66" op: "Sub" input: "bert/encoder/layer_3/output/dense/kernel/read" input: "mul_360" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Assign_401" op: "Assign" input: "bert/encoder/layer_3/output/dense/kernel" input: "sub_66" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_402" op: "Assign" input: "bert/encoder/layer_3/output/dense/kernel/adam_m" input: "add_224" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_403" op: "Assign" input: "bert/encoder/layer_3/output/dense/kernel/adam_v" input: "add_225" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/output/dense/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/output/dense/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_3/output/dense/bias/adam_m" input: "bert/encoder/layer_3/output/dense/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/output/dense/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_3/output/dense/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/output/dense/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/output/dense/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_3/output/dense/bias/adam_v" input: "bert/encoder/layer_3/output/dense/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/output/dense/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_3/output/dense/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_361/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_361" op: "Mul" input: "Mul_361/x" input: "bert/encoder/layer_3/output/dense/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_362/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_362" op: "Mul" input: "Mul_362/x" input: "clip_by_global_norm/clip_by_global_norm/_66" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_228" op: "Add" input: "Mul_361" input: "Mul_362" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_363/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_363" op: "Mul" input: "Mul_363/x" input: "bert/encoder/layer_3/output/dense/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_66" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_66" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_364/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_364" op: "Mul" input: "Mul_364/x" input: "Square_66" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_229" op: "Add" input: "Mul_363" input: "Mul_364" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_66" op: "Sqrt" input: "add_229" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_230/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_230" op: "Add" input: "Sqrt_66" input: "add_230/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_67" op: "RealDiv" input: "add_228" input: "add_230" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_365" op: "Mul" input: "add_2" input: "truediv_67" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_67" op: "Sub" input: "bert/encoder/layer_3/output/dense/bias/read" input: "mul_365" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_404" op: "Assign" input: "bert/encoder/layer_3/output/dense/bias" input: "sub_67" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_405" op: "Assign" input: "bert/encoder/layer_3/output/dense/bias/adam_m" input: "add_228" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_406" op: "Assign" input: "bert/encoder/layer_3/output/dense/bias/adam_v" input: "add_229" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/output/LayerNorm/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/output/LayerNorm/beta/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/output/LayerNorm/beta/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_3/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_3/output/LayerNorm/beta/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/output/LayerNorm/beta/adam_m/read" op: "Identity" input: "bert/encoder/layer_3/output/LayerNorm/beta/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/output/LayerNorm/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/output/LayerNorm/beta/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/output/LayerNorm/beta/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_3/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_3/output/LayerNorm/beta/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/output/LayerNorm/beta/adam_v/read" op: "Identity" input: "bert/encoder/layer_3/output/LayerNorm/beta/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_366/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_366" op: "Mul" input: "Mul_366/x" input: "bert/encoder/layer_3/output/LayerNorm/beta/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_367/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_367" op: "Mul" input: "Mul_367/x" input: "clip_by_global_norm/clip_by_global_norm/_67" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_231" op: "Add" input: "Mul_366" input: "Mul_367" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_368/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_368" op: "Mul" input: "Mul_368/x" input: "bert/encoder/layer_3/output/LayerNorm/beta/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_67" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_67" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_369/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_369" op: "Mul" input: "Mul_369/x" input: "Square_67" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_232" op: "Add" input: "Mul_368" input: "Mul_369" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_67" op: "Sqrt" input: "add_232" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_233/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_233" op: "Add" input: "Sqrt_67" input: "add_233/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_68" op: "RealDiv" input: "add_231" input: "add_233" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_370" op: "Mul" input: "add_2" input: "truediv_68" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_68" op: "Sub" input: "bert/encoder/layer_3/output/LayerNorm/beta/read" input: "mul_370" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_407" op: "Assign" input: "bert/encoder/layer_3/output/LayerNorm/beta" input: "sub_68" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_408" op: "Assign" input: "bert/encoder/layer_3/output/LayerNorm/beta/adam_m" input: "add_231" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_409" op: "Assign" input: "bert/encoder/layer_3/output/LayerNorm/beta/adam_v" input: "add_232" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/output/LayerNorm/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/output/LayerNorm/gamma/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/output/LayerNorm/gamma/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_3/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_3/output/LayerNorm/gamma/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/output/LayerNorm/gamma/adam_m/read" op: "Identity" input: "bert/encoder/layer_3/output/LayerNorm/gamma/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/output/LayerNorm/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/output/LayerNorm/gamma/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_3/output/LayerNorm/gamma/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_3/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_3/output/LayerNorm/gamma/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_3/output/LayerNorm/gamma/adam_v/read" op: "Identity" input: "bert/encoder/layer_3/output/LayerNorm/gamma/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_371/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_371" op: "Mul" input: "Mul_371/x" input: "bert/encoder/layer_3/output/LayerNorm/gamma/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_372/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_372" op: "Mul" input: "Mul_372/x" input: "clip_by_global_norm/clip_by_global_norm/_68" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_234" op: "Add" input: "Mul_371" input: "Mul_372" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_373/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_373" op: "Mul" input: "Mul_373/x" input: "bert/encoder/layer_3/output/LayerNorm/gamma/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_68" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_68" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_374/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_374" op: "Mul" input: "Mul_374/x" input: "Square_68" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_235" op: "Add" input: "Mul_373" input: "Mul_374" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_68" op: "Sqrt" input: "add_235" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_236/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_236" op: "Add" input: "Sqrt_68" input: "add_236/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_69" op: "RealDiv" input: "add_234" input: "add_236" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_375" op: "Mul" input: "add_2" input: "truediv_69" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_69" op: "Sub" input: "bert/encoder/layer_3/output/LayerNorm/gamma/read" input: "mul_375" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_410" op: "Assign" input: "bert/encoder/layer_3/output/LayerNorm/gamma" input: "sub_69" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_411" op: "Assign" input: "bert/encoder/layer_3/output/LayerNorm/gamma/adam_m" input: "add_234" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_412" op: "Assign" input: "bert/encoder/layer_3/output/LayerNorm/gamma/adam_v" input: "add_235" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_m" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_v" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_376/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_376" op: "Mul" input: "Mul_376/x" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_377/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_377" op: "Mul" input: "Mul_377/x" input: "clip_by_global_norm/clip_by_global_norm/_69" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_237" op: "Add" input: "Mul_376" input: "Mul_377" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_378/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_378" op: "Mul" input: "Mul_378/x" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_69" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_69" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_379/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_379" op: "Mul" input: "Mul_379/x" input: "Square_69" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_238" op: "Add" input: "Mul_378" input: "Mul_379" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_69" op: "Sqrt" input: "add_238" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_239/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_239" op: "Add" input: "Sqrt_69" input: "add_239/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_70" op: "RealDiv" input: "add_237" input: "add_239" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_380/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_380" op: "Mul" input: "mul_380/x" input: "bert/encoder/layer_4/attention/self/query/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_240" op: "Add" input: "truediv_70" input: "mul_380" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_381" op: "Mul" input: "add_2" input: "add_240" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_70" op: "Sub" input: "bert/encoder/layer_4/attention/self/query/kernel/read" input: "mul_381" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_413" op: "Assign" input: "bert/encoder/layer_4/attention/self/query/kernel" input: "sub_70" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_414" op: "Assign" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_m" input: "add_237" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_415" op: "Assign" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_v" input: "add_238" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/attention/self/query/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/self/query/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/attention/self/query/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_4/attention/self/query/bias/adam_m" input: "bert/encoder/layer_4/attention/self/query/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/attention/self/query/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_4/attention/self/query/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/self/query/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/self/query/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/attention/self/query/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_4/attention/self/query/bias/adam_v" input: "bert/encoder/layer_4/attention/self/query/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/attention/self/query/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_4/attention/self/query/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_382/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_382" op: "Mul" input: "Mul_382/x" input: "bert/encoder/layer_4/attention/self/query/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_383/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_383" op: "Mul" input: "Mul_383/x" input: "clip_by_global_norm/clip_by_global_norm/_70" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_241" op: "Add" input: "Mul_382" input: "Mul_383" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_384/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_384" op: "Mul" input: "Mul_384/x" input: "bert/encoder/layer_4/attention/self/query/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_70" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_70" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_385/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_385" op: "Mul" input: "Mul_385/x" input: "Square_70" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_242" op: "Add" input: "Mul_384" input: "Mul_385" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_70" op: "Sqrt" input: "add_242" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_243/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_243" op: "Add" input: "Sqrt_70" input: "add_243/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_71" op: "RealDiv" input: "add_241" input: "add_243" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_386" op: "Mul" input: "add_2" input: "truediv_71" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_71" op: "Sub" input: "bert/encoder/layer_4/attention/self/query/bias/read" input: "mul_386" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_416" op: "Assign" input: "bert/encoder/layer_4/attention/self/query/bias" input: "sub_71" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_417" op: "Assign" input: "bert/encoder/layer_4/attention/self/query/bias/adam_m" input: "add_241" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_418" op: "Assign" input: "bert/encoder/layer_4/attention/self/query/bias/adam_v" input: "add_242" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_m" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_v" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_387/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_387" op: "Mul" input: "Mul_387/x" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_388/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_388" op: "Mul" input: "Mul_388/x" input: "clip_by_global_norm/clip_by_global_norm/_71" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_244" op: "Add" input: "Mul_387" input: "Mul_388" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_389/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_389" op: "Mul" input: "Mul_389/x" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_71" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_71" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_390/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_390" op: "Mul" input: "Mul_390/x" input: "Square_71" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_245" op: "Add" input: "Mul_389" input: "Mul_390" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_71" op: "Sqrt" input: "add_245" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_246/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_246" op: "Add" input: "Sqrt_71" input: "add_246/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_72" op: "RealDiv" input: "add_244" input: "add_246" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_391/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_391" op: "Mul" input: "mul_391/x" input: "bert/encoder/layer_4/attention/self/key/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_247" op: "Add" input: "truediv_72" input: "mul_391" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_392" op: "Mul" input: "add_2" input: "add_247" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_72" op: "Sub" input: "bert/encoder/layer_4/attention/self/key/kernel/read" input: "mul_392" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_419" op: "Assign" input: "bert/encoder/layer_4/attention/self/key/kernel" input: "sub_72" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_420" op: "Assign" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_m" input: "add_244" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_421" op: "Assign" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_v" input: "add_245" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/attention/self/key/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/self/key/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/attention/self/key/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_4/attention/self/key/bias/adam_m" input: "bert/encoder/layer_4/attention/self/key/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/attention/self/key/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_4/attention/self/key/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/self/key/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/self/key/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/attention/self/key/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_4/attention/self/key/bias/adam_v" input: "bert/encoder/layer_4/attention/self/key/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/attention/self/key/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_4/attention/self/key/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_393/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_393" op: "Mul" input: "Mul_393/x" input: "bert/encoder/layer_4/attention/self/key/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_394/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_394" op: "Mul" input: "Mul_394/x" input: "clip_by_global_norm/clip_by_global_norm/_72" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_248" op: "Add" input: "Mul_393" input: "Mul_394" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_395/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_395" op: "Mul" input: "Mul_395/x" input: "bert/encoder/layer_4/attention/self/key/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_72" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_72" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_396/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_396" op: "Mul" input: "Mul_396/x" input: "Square_72" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_249" op: "Add" input: "Mul_395" input: "Mul_396" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_72" op: "Sqrt" input: "add_249" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_250/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_250" op: "Add" input: "Sqrt_72" input: "add_250/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_73" op: "RealDiv" input: "add_248" input: "add_250" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_397" op: "Mul" input: "add_2" input: "truediv_73" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_73" op: "Sub" input: "bert/encoder/layer_4/attention/self/key/bias/read" input: "mul_397" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_422" op: "Assign" input: "bert/encoder/layer_4/attention/self/key/bias" input: "sub_73" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_423" op: "Assign" input: "bert/encoder/layer_4/attention/self/key/bias/adam_m" input: "add_248" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_424" op: "Assign" input: "bert/encoder/layer_4/attention/self/key/bias/adam_v" input: "add_249" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_m" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_v" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_398/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_398" op: "Mul" input: "Mul_398/x" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_399/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_399" op: "Mul" input: "Mul_399/x" input: "clip_by_global_norm/clip_by_global_norm/_73" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_251" op: "Add" input: "Mul_398" input: "Mul_399" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_400/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_400" op: "Mul" input: "Mul_400/x" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_73" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_73" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_401/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_401" op: "Mul" input: "Mul_401/x" input: "Square_73" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_252" op: "Add" input: "Mul_400" input: "Mul_401" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_73" op: "Sqrt" input: "add_252" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_253/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_253" op: "Add" input: "Sqrt_73" input: "add_253/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_74" op: "RealDiv" input: "add_251" input: "add_253" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_402/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_402" op: "Mul" input: "mul_402/x" input: "bert/encoder/layer_4/attention/self/value/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_254" op: "Add" input: "truediv_74" input: "mul_402" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_403" op: "Mul" input: "add_2" input: "add_254" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_74" op: "Sub" input: "bert/encoder/layer_4/attention/self/value/kernel/read" input: "mul_403" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_425" op: "Assign" input: "bert/encoder/layer_4/attention/self/value/kernel" input: "sub_74" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_426" op: "Assign" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_m" input: "add_251" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_427" op: "Assign" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_v" input: "add_252" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/attention/self/value/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/self/value/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/attention/self/value/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_4/attention/self/value/bias/adam_m" input: "bert/encoder/layer_4/attention/self/value/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/attention/self/value/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_4/attention/self/value/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/self/value/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/self/value/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/attention/self/value/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_4/attention/self/value/bias/adam_v" input: "bert/encoder/layer_4/attention/self/value/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/attention/self/value/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_4/attention/self/value/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_404/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_404" op: "Mul" input: "Mul_404/x" input: "bert/encoder/layer_4/attention/self/value/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_405/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_405" op: "Mul" input: "Mul_405/x" input: "clip_by_global_norm/clip_by_global_norm/_74" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_255" op: "Add" input: "Mul_404" input: "Mul_405" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_406/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_406" op: "Mul" input: "Mul_406/x" input: "bert/encoder/layer_4/attention/self/value/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_74" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_74" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_407/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_407" op: "Mul" input: "Mul_407/x" input: "Square_74" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_256" op: "Add" input: "Mul_406" input: "Mul_407" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_74" op: "Sqrt" input: "add_256" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_257/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_257" op: "Add" input: "Sqrt_74" input: "add_257/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_75" op: "RealDiv" input: "add_255" input: "add_257" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_408" op: "Mul" input: "add_2" input: "truediv_75" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_75" op: "Sub" input: "bert/encoder/layer_4/attention/self/value/bias/read" input: "mul_408" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_428" op: "Assign" input: "bert/encoder/layer_4/attention/self/value/bias" input: "sub_75" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_429" op: "Assign" input: "bert/encoder/layer_4/attention/self/value/bias/adam_m" input: "add_255" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_430" op: "Assign" input: "bert/encoder/layer_4/attention/self/value/bias/adam_v" input: "add_256" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_409/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_409" op: "Mul" input: "Mul_409/x" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_410/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_410" op: "Mul" input: "Mul_410/x" input: "clip_by_global_norm/clip_by_global_norm/_75" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_258" op: "Add" input: "Mul_409" input: "Mul_410" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_411/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_411" op: "Mul" input: "Mul_411/x" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_75" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_75" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_412/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_412" op: "Mul" input: "Mul_412/x" input: "Square_75" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_259" op: "Add" input: "Mul_411" input: "Mul_412" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_75" op: "Sqrt" input: "add_259" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_260/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_260" op: "Add" input: "Sqrt_75" input: "add_260/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_76" op: "RealDiv" input: "add_258" input: "add_260" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_413/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_413" op: "Mul" input: "mul_413/x" input: "bert/encoder/layer_4/attention/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_261" op: "Add" input: "truediv_76" input: "mul_413" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_414" op: "Mul" input: "add_2" input: "add_261" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_76" op: "Sub" input: "bert/encoder/layer_4/attention/output/dense/kernel/read" input: "mul_414" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_431" op: "Assign" input: "bert/encoder/layer_4/attention/output/dense/kernel" input: "sub_76" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_432" op: "Assign" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m" input: "add_258" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_433" op: "Assign" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v" input: "add_259" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/attention/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/output/dense/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/attention/output/dense/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_4/attention/output/dense/bias/adam_m" input: "bert/encoder/layer_4/attention/output/dense/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/attention/output/dense/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_4/attention/output/dense/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/output/dense/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/attention/output/dense/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_4/attention/output/dense/bias/adam_v" input: "bert/encoder/layer_4/attention/output/dense/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/attention/output/dense/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_4/attention/output/dense/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_415/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_415" op: "Mul" input: "Mul_415/x" input: "bert/encoder/layer_4/attention/output/dense/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_416/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_416" op: "Mul" input: "Mul_416/x" input: "clip_by_global_norm/clip_by_global_norm/_76" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_262" op: "Add" input: "Mul_415" input: "Mul_416" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_417/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_417" op: "Mul" input: "Mul_417/x" input: "bert/encoder/layer_4/attention/output/dense/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_76" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_76" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_418/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_418" op: "Mul" input: "Mul_418/x" input: "Square_76" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_263" op: "Add" input: "Mul_417" input: "Mul_418" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_76" op: "Sqrt" input: "add_263" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_264/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_264" op: "Add" input: "Sqrt_76" input: "add_264/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_77" op: "RealDiv" input: "add_262" input: "add_264" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_419" op: "Mul" input: "add_2" input: "truediv_77" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_77" op: "Sub" input: "bert/encoder/layer_4/attention/output/dense/bias/read" input: "mul_419" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_434" op: "Assign" input: "bert/encoder/layer_4/attention/output/dense/bias" input: "sub_77" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_435" op: "Assign" input: "bert/encoder/layer_4/attention/output/dense/bias/adam_m" input: "add_262" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_436" op: "Assign" input: "bert/encoder/layer_4/attention/output/dense/bias/adam_v" input: "add_263" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_m/read" op: "Identity" input: "bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_v/read" op: "Identity" input: "bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_420/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_420" op: "Mul" input: "Mul_420/x" input: "bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_421/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_421" op: "Mul" input: "Mul_421/x" input: "clip_by_global_norm/clip_by_global_norm/_77" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_265" op: "Add" input: "Mul_420" input: "Mul_421" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_422/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_422" op: "Mul" input: "Mul_422/x" input: "bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_77" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_77" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_423/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_423" op: "Mul" input: "Mul_423/x" input: "Square_77" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_266" op: "Add" input: "Mul_422" input: "Mul_423" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_77" op: "Sqrt" input: "add_266" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_267/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_267" op: "Add" input: "Sqrt_77" input: "add_267/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_78" op: "RealDiv" input: "add_265" input: "add_267" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_424" op: "Mul" input: "add_2" input: "truediv_78" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_78" op: "Sub" input: "bert/encoder/layer_4/attention/output/LayerNorm/beta/read" input: "mul_424" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_437" op: "Assign" input: "bert/encoder/layer_4/attention/output/LayerNorm/beta" input: "sub_78" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_438" op: "Assign" input: "bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_m" input: "add_265" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_439" op: "Assign" input: "bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_v" input: "add_266" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m/read" op: "Identity" input: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v/read" op: "Identity" input: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_425/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_425" op: "Mul" input: "Mul_425/x" input: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_426/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_426" op: "Mul" input: "Mul_426/x" input: "clip_by_global_norm/clip_by_global_norm/_78" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_268" op: "Add" input: "Mul_425" input: "Mul_426" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_427/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_427" op: "Mul" input: "Mul_427/x" input: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_78" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_78" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_428/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_428" op: "Mul" input: "Mul_428/x" input: "Square_78" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_269" op: "Add" input: "Mul_427" input: "Mul_428" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_78" op: "Sqrt" input: "add_269" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_270/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_270" op: "Add" input: "Sqrt_78" input: "add_270/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_79" op: "RealDiv" input: "add_268" input: "add_270" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_429" op: "Mul" input: "add_2" input: "truediv_79" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_79" op: "Sub" input: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/read" input: "mul_429" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_440" op: "Assign" input: "bert/encoder/layer_4/attention/output/LayerNorm/gamma" input: "sub_79" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_441" op: "Assign" input: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m" input: "add_268" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_442" op: "Assign" input: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v" input: "add_269" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_430/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_430" op: "Mul" input: "Mul_430/x" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_431/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_431" op: "Mul" input: "Mul_431/x" input: "clip_by_global_norm/clip_by_global_norm/_79" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_271" op: "Add" input: "Mul_430" input: "Mul_431" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_432/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_432" op: "Mul" input: "Mul_432/x" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Square_79" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_79" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_433/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_433" op: "Mul" input: "Mul_433/x" input: "Square_79" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_272" op: "Add" input: "Mul_432" input: "Mul_433" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Sqrt_79" op: "Sqrt" input: "add_272" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_273/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_273" op: "Add" input: "Sqrt_79" input: "add_273/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "truediv_80" op: "RealDiv" input: "add_271" input: "add_273" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "mul_434/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_434" op: "Mul" input: "mul_434/x" input: "bert/encoder/layer_4/intermediate/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_274" op: "Add" input: "truediv_80" input: "mul_434" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "mul_435" op: "Mul" input: "add_2" input: "add_274" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "sub_80" op: "Sub" input: "bert/encoder/layer_4/intermediate/dense/kernel/read" input: "mul_435" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Assign_443" op: "Assign" input: "bert/encoder/layer_4/intermediate/dense/kernel" input: "sub_80" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_444" op: "Assign" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m" input: "add_271" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_445" op: "Assign" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v" input: "add_272" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_4/intermediate/dense/bias/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/intermediate/dense/bias/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_4/intermediate/dense/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/intermediate/dense/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_m" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/intermediate/dense/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_4/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_4/intermediate/dense/bias/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/intermediate/dense/bias/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_4/intermediate/dense/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/intermediate/dense/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_v" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/intermediate/dense/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_436/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_436" op: "Mul" input: "Mul_436/x" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_437/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_437" op: "Mul" input: "Mul_437/x" input: "clip_by_global_norm/clip_by_global_norm/_80" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_275" op: "Add" input: "Mul_436" input: "Mul_437" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_438/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_438" op: "Mul" input: "Mul_438/x" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Square_80" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_80" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_439/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_439" op: "Mul" input: "Mul_439/x" input: "Square_80" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_276" op: "Add" input: "Mul_438" input: "Mul_439" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Sqrt_80" op: "Sqrt" input: "add_276" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_277/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_277" op: "Add" input: "Sqrt_80" input: "add_277/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "truediv_81" op: "RealDiv" input: "add_275" input: "add_277" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "mul_440" op: "Mul" input: "add_2" input: "truediv_81" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "sub_81" op: "Sub" input: "bert/encoder/layer_4/intermediate/dense/bias/read" input: "mul_440" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Assign_446" op: "Assign" input: "bert/encoder/layer_4/intermediate/dense/bias" input: "sub_81" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_447" op: "Assign" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_m" input: "add_275" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_448" op: "Assign" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_v" input: "add_276" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_4/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_4/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_4/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_4/output/dense/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/output/dense/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_4/output/dense/kernel/adam_m" input: "bert/encoder/layer_4/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/output/dense/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_4/output/dense/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_4/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_4/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_4/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_4/output/dense/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/output/dense/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_4/output/dense/kernel/adam_v" input: "bert/encoder/layer_4/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/output/dense/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_4/output/dense/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_441/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_441" op: "Mul" input: "Mul_441/x" input: "bert/encoder/layer_4/output/dense/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_442/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_442" op: "Mul" input: "Mul_442/x" input: "clip_by_global_norm/clip_by_global_norm/_81" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_278" op: "Add" input: "Mul_441" input: "Mul_442" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_443/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_443" op: "Mul" input: "Mul_443/x" input: "bert/encoder/layer_4/output/dense/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Square_81" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_81" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_444/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_444" op: "Mul" input: "Mul_444/x" input: "Square_81" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_279" op: "Add" input: "Mul_443" input: "Mul_444" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Sqrt_81" op: "Sqrt" input: "add_279" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_280/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_280" op: "Add" input: "Sqrt_81" input: "add_280/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "truediv_82" op: "RealDiv" input: "add_278" input: "add_280" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "mul_445/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_445" op: "Mul" input: "mul_445/x" input: "bert/encoder/layer_4/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_281" op: "Add" input: "truediv_82" input: "mul_445" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "mul_446" op: "Mul" input: "add_2" input: "add_281" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "sub_82" op: "Sub" input: "bert/encoder/layer_4/output/dense/kernel/read" input: "mul_446" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Assign_449" op: "Assign" input: "bert/encoder/layer_4/output/dense/kernel" input: "sub_82" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_450" op: "Assign" input: "bert/encoder/layer_4/output/dense/kernel/adam_m" input: "add_278" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_451" op: "Assign" input: "bert/encoder/layer_4/output/dense/kernel/adam_v" input: "add_279" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/output/dense/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/output/dense/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_4/output/dense/bias/adam_m" input: "bert/encoder/layer_4/output/dense/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/output/dense/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_4/output/dense/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/output/dense/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/output/dense/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_4/output/dense/bias/adam_v" input: "bert/encoder/layer_4/output/dense/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/output/dense/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_4/output/dense/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_447/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_447" op: "Mul" input: "Mul_447/x" input: "bert/encoder/layer_4/output/dense/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_448/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_448" op: "Mul" input: "Mul_448/x" input: "clip_by_global_norm/clip_by_global_norm/_82" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_282" op: "Add" input: "Mul_447" input: "Mul_448" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_449/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_449" op: "Mul" input: "Mul_449/x" input: "bert/encoder/layer_4/output/dense/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_82" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_82" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_450/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_450" op: "Mul" input: "Mul_450/x" input: "Square_82" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_283" op: "Add" input: "Mul_449" input: "Mul_450" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_82" op: "Sqrt" input: "add_283" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_284/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_284" op: "Add" input: "Sqrt_82" input: "add_284/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_83" op: "RealDiv" input: "add_282" input: "add_284" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_451" op: "Mul" input: "add_2" input: "truediv_83" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_83" op: "Sub" input: "bert/encoder/layer_4/output/dense/bias/read" input: "mul_451" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_452" op: "Assign" input: "bert/encoder/layer_4/output/dense/bias" input: "sub_83" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_453" op: "Assign" input: "bert/encoder/layer_4/output/dense/bias/adam_m" input: "add_282" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_454" op: "Assign" input: "bert/encoder/layer_4/output/dense/bias/adam_v" input: "add_283" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/output/LayerNorm/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/output/LayerNorm/beta/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/output/LayerNorm/beta/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_4/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_4/output/LayerNorm/beta/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/output/LayerNorm/beta/adam_m/read" op: "Identity" input: "bert/encoder/layer_4/output/LayerNorm/beta/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/output/LayerNorm/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/output/LayerNorm/beta/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/output/LayerNorm/beta/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_4/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_4/output/LayerNorm/beta/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/output/LayerNorm/beta/adam_v/read" op: "Identity" input: "bert/encoder/layer_4/output/LayerNorm/beta/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_452/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_452" op: "Mul" input: "Mul_452/x" input: "bert/encoder/layer_4/output/LayerNorm/beta/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_453/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_453" op: "Mul" input: "Mul_453/x" input: "clip_by_global_norm/clip_by_global_norm/_83" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_285" op: "Add" input: "Mul_452" input: "Mul_453" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_454/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_454" op: "Mul" input: "Mul_454/x" input: "bert/encoder/layer_4/output/LayerNorm/beta/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_83" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_83" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_455/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_455" op: "Mul" input: "Mul_455/x" input: "Square_83" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_286" op: "Add" input: "Mul_454" input: "Mul_455" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_83" op: "Sqrt" input: "add_286" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_287/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_287" op: "Add" input: "Sqrt_83" input: "add_287/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_84" op: "RealDiv" input: "add_285" input: "add_287" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_456" op: "Mul" input: "add_2" input: "truediv_84" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_84" op: "Sub" input: "bert/encoder/layer_4/output/LayerNorm/beta/read" input: "mul_456" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_455" op: "Assign" input: "bert/encoder/layer_4/output/LayerNorm/beta" input: "sub_84" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_456" op: "Assign" input: "bert/encoder/layer_4/output/LayerNorm/beta/adam_m" input: "add_285" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_457" op: "Assign" input: "bert/encoder/layer_4/output/LayerNorm/beta/adam_v" input: "add_286" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/output/LayerNorm/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/output/LayerNorm/gamma/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/output/LayerNorm/gamma/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_4/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_4/output/LayerNorm/gamma/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/output/LayerNorm/gamma/adam_m/read" op: "Identity" input: "bert/encoder/layer_4/output/LayerNorm/gamma/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/output/LayerNorm/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/output/LayerNorm/gamma/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_4/output/LayerNorm/gamma/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_4/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_4/output/LayerNorm/gamma/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_4/output/LayerNorm/gamma/adam_v/read" op: "Identity" input: "bert/encoder/layer_4/output/LayerNorm/gamma/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_457/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_457" op: "Mul" input: "Mul_457/x" input: "bert/encoder/layer_4/output/LayerNorm/gamma/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_458/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_458" op: "Mul" input: "Mul_458/x" input: "clip_by_global_norm/clip_by_global_norm/_84" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_288" op: "Add" input: "Mul_457" input: "Mul_458" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_459/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_459" op: "Mul" input: "Mul_459/x" input: "bert/encoder/layer_4/output/LayerNorm/gamma/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_84" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_84" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_460/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_460" op: "Mul" input: "Mul_460/x" input: "Square_84" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_289" op: "Add" input: "Mul_459" input: "Mul_460" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_84" op: "Sqrt" input: "add_289" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_290/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_290" op: "Add" input: "Sqrt_84" input: "add_290/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_85" op: "RealDiv" input: "add_288" input: "add_290" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_461" op: "Mul" input: "add_2" input: "truediv_85" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_85" op: "Sub" input: "bert/encoder/layer_4/output/LayerNorm/gamma/read" input: "mul_461" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_458" op: "Assign" input: "bert/encoder/layer_4/output/LayerNorm/gamma" input: "sub_85" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_459" op: "Assign" input: "bert/encoder/layer_4/output/LayerNorm/gamma/adam_m" input: "add_288" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_460" op: "Assign" input: "bert/encoder/layer_4/output/LayerNorm/gamma/adam_v" input: "add_289" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_m" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_v" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_462/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_462" op: "Mul" input: "Mul_462/x" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_463/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_463" op: "Mul" input: "Mul_463/x" input: "clip_by_global_norm/clip_by_global_norm/_85" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_291" op: "Add" input: "Mul_462" input: "Mul_463" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_464/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_464" op: "Mul" input: "Mul_464/x" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_85" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_85" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_465/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_465" op: "Mul" input: "Mul_465/x" input: "Square_85" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_292" op: "Add" input: "Mul_464" input: "Mul_465" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_85" op: "Sqrt" input: "add_292" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_293/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_293" op: "Add" input: "Sqrt_85" input: "add_293/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_86" op: "RealDiv" input: "add_291" input: "add_293" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_466/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_466" op: "Mul" input: "mul_466/x" input: "bert/encoder/layer_5/attention/self/query/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_294" op: "Add" input: "truediv_86" input: "mul_466" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_467" op: "Mul" input: "add_2" input: "add_294" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_86" op: "Sub" input: "bert/encoder/layer_5/attention/self/query/kernel/read" input: "mul_467" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_461" op: "Assign" input: "bert/encoder/layer_5/attention/self/query/kernel" input: "sub_86" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_462" op: "Assign" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_m" input: "add_291" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_463" op: "Assign" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_v" input: "add_292" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/attention/self/query/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/self/query/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/attention/self/query/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_5/attention/self/query/bias/adam_m" input: "bert/encoder/layer_5/attention/self/query/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/attention/self/query/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_5/attention/self/query/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/self/query/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/self/query/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/attention/self/query/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_5/attention/self/query/bias/adam_v" input: "bert/encoder/layer_5/attention/self/query/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/attention/self/query/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_5/attention/self/query/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_468/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_468" op: "Mul" input: "Mul_468/x" input: "bert/encoder/layer_5/attention/self/query/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_469/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_469" op: "Mul" input: "Mul_469/x" input: "clip_by_global_norm/clip_by_global_norm/_86" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_295" op: "Add" input: "Mul_468" input: "Mul_469" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_470/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_470" op: "Mul" input: "Mul_470/x" input: "bert/encoder/layer_5/attention/self/query/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_86" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_86" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_471/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_471" op: "Mul" input: "Mul_471/x" input: "Square_86" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_296" op: "Add" input: "Mul_470" input: "Mul_471" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_86" op: "Sqrt" input: "add_296" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_297/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_297" op: "Add" input: "Sqrt_86" input: "add_297/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_87" op: "RealDiv" input: "add_295" input: "add_297" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_472" op: "Mul" input: "add_2" input: "truediv_87" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_87" op: "Sub" input: "bert/encoder/layer_5/attention/self/query/bias/read" input: "mul_472" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_464" op: "Assign" input: "bert/encoder/layer_5/attention/self/query/bias" input: "sub_87" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_465" op: "Assign" input: "bert/encoder/layer_5/attention/self/query/bias/adam_m" input: "add_295" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_466" op: "Assign" input: "bert/encoder/layer_5/attention/self/query/bias/adam_v" input: "add_296" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_m" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_v" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_473/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_473" op: "Mul" input: "Mul_473/x" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_474/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_474" op: "Mul" input: "Mul_474/x" input: "clip_by_global_norm/clip_by_global_norm/_87" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_298" op: "Add" input: "Mul_473" input: "Mul_474" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_475/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_475" op: "Mul" input: "Mul_475/x" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_87" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_87" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_476/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_476" op: "Mul" input: "Mul_476/x" input: "Square_87" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_299" op: "Add" input: "Mul_475" input: "Mul_476" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_87" op: "Sqrt" input: "add_299" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_300/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_300" op: "Add" input: "Sqrt_87" input: "add_300/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_88" op: "RealDiv" input: "add_298" input: "add_300" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_477/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_477" op: "Mul" input: "mul_477/x" input: "bert/encoder/layer_5/attention/self/key/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_301" op: "Add" input: "truediv_88" input: "mul_477" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_478" op: "Mul" input: "add_2" input: "add_301" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_88" op: "Sub" input: "bert/encoder/layer_5/attention/self/key/kernel/read" input: "mul_478" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_467" op: "Assign" input: "bert/encoder/layer_5/attention/self/key/kernel" input: "sub_88" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_468" op: "Assign" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_m" input: "add_298" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_469" op: "Assign" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_v" input: "add_299" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/attention/self/key/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/self/key/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/attention/self/key/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_5/attention/self/key/bias/adam_m" input: "bert/encoder/layer_5/attention/self/key/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/attention/self/key/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_5/attention/self/key/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/self/key/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/self/key/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/attention/self/key/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_5/attention/self/key/bias/adam_v" input: "bert/encoder/layer_5/attention/self/key/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/attention/self/key/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_5/attention/self/key/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_479/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_479" op: "Mul" input: "Mul_479/x" input: "bert/encoder/layer_5/attention/self/key/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_480/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_480" op: "Mul" input: "Mul_480/x" input: "clip_by_global_norm/clip_by_global_norm/_88" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_302" op: "Add" input: "Mul_479" input: "Mul_480" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_481/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_481" op: "Mul" input: "Mul_481/x" input: "bert/encoder/layer_5/attention/self/key/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_88" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_88" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_482/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_482" op: "Mul" input: "Mul_482/x" input: "Square_88" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_303" op: "Add" input: "Mul_481" input: "Mul_482" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_88" op: "Sqrt" input: "add_303" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_304/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_304" op: "Add" input: "Sqrt_88" input: "add_304/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_89" op: "RealDiv" input: "add_302" input: "add_304" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_483" op: "Mul" input: "add_2" input: "truediv_89" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_89" op: "Sub" input: "bert/encoder/layer_5/attention/self/key/bias/read" input: "mul_483" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_470" op: "Assign" input: "bert/encoder/layer_5/attention/self/key/bias" input: "sub_89" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_471" op: "Assign" input: "bert/encoder/layer_5/attention/self/key/bias/adam_m" input: "add_302" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_472" op: "Assign" input: "bert/encoder/layer_5/attention/self/key/bias/adam_v" input: "add_303" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_m" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_v" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_484/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_484" op: "Mul" input: "Mul_484/x" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_485/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_485" op: "Mul" input: "Mul_485/x" input: "clip_by_global_norm/clip_by_global_norm/_89" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_305" op: "Add" input: "Mul_484" input: "Mul_485" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_486/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_486" op: "Mul" input: "Mul_486/x" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_89" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_89" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_487/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_487" op: "Mul" input: "Mul_487/x" input: "Square_89" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_306" op: "Add" input: "Mul_486" input: "Mul_487" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_89" op: "Sqrt" input: "add_306" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_307/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_307" op: "Add" input: "Sqrt_89" input: "add_307/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_90" op: "RealDiv" input: "add_305" input: "add_307" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_488/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_488" op: "Mul" input: "mul_488/x" input: "bert/encoder/layer_5/attention/self/value/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_308" op: "Add" input: "truediv_90" input: "mul_488" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_489" op: "Mul" input: "add_2" input: "add_308" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_90" op: "Sub" input: "bert/encoder/layer_5/attention/self/value/kernel/read" input: "mul_489" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_473" op: "Assign" input: "bert/encoder/layer_5/attention/self/value/kernel" input: "sub_90" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_474" op: "Assign" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_m" input: "add_305" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_475" op: "Assign" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_v" input: "add_306" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/attention/self/value/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/self/value/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/attention/self/value/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_5/attention/self/value/bias/adam_m" input: "bert/encoder/layer_5/attention/self/value/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/attention/self/value/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_5/attention/self/value/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/self/value/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/self/value/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/attention/self/value/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_5/attention/self/value/bias/adam_v" input: "bert/encoder/layer_5/attention/self/value/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/attention/self/value/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_5/attention/self/value/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_490/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_490" op: "Mul" input: "Mul_490/x" input: "bert/encoder/layer_5/attention/self/value/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_491/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_491" op: "Mul" input: "Mul_491/x" input: "clip_by_global_norm/clip_by_global_norm/_90" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_309" op: "Add" input: "Mul_490" input: "Mul_491" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_492/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_492" op: "Mul" input: "Mul_492/x" input: "bert/encoder/layer_5/attention/self/value/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_90" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_90" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_493/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_493" op: "Mul" input: "Mul_493/x" input: "Square_90" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_310" op: "Add" input: "Mul_492" input: "Mul_493" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_90" op: "Sqrt" input: "add_310" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_311/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_311" op: "Add" input: "Sqrt_90" input: "add_311/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_91" op: "RealDiv" input: "add_309" input: "add_311" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_494" op: "Mul" input: "add_2" input: "truediv_91" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_91" op: "Sub" input: "bert/encoder/layer_5/attention/self/value/bias/read" input: "mul_494" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_476" op: "Assign" input: "bert/encoder/layer_5/attention/self/value/bias" input: "sub_91" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_477" op: "Assign" input: "bert/encoder/layer_5/attention/self/value/bias/adam_m" input: "add_309" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_478" op: "Assign" input: "bert/encoder/layer_5/attention/self/value/bias/adam_v" input: "add_310" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_495/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_495" op: "Mul" input: "Mul_495/x" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_496/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_496" op: "Mul" input: "Mul_496/x" input: "clip_by_global_norm/clip_by_global_norm/_91" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_312" op: "Add" input: "Mul_495" input: "Mul_496" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_497/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_497" op: "Mul" input: "Mul_497/x" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_91" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_91" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_498/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_498" op: "Mul" input: "Mul_498/x" input: "Square_91" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_313" op: "Add" input: "Mul_497" input: "Mul_498" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_91" op: "Sqrt" input: "add_313" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_314/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_314" op: "Add" input: "Sqrt_91" input: "add_314/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_92" op: "RealDiv" input: "add_312" input: "add_314" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_499/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_499" op: "Mul" input: "mul_499/x" input: "bert/encoder/layer_5/attention/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_315" op: "Add" input: "truediv_92" input: "mul_499" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_500" op: "Mul" input: "add_2" input: "add_315" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_92" op: "Sub" input: "bert/encoder/layer_5/attention/output/dense/kernel/read" input: "mul_500" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_479" op: "Assign" input: "bert/encoder/layer_5/attention/output/dense/kernel" input: "sub_92" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_480" op: "Assign" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m" input: "add_312" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_481" op: "Assign" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v" input: "add_313" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/attention/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/output/dense/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/attention/output/dense/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_5/attention/output/dense/bias/adam_m" input: "bert/encoder/layer_5/attention/output/dense/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/attention/output/dense/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_5/attention/output/dense/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/output/dense/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/attention/output/dense/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_5/attention/output/dense/bias/adam_v" input: "bert/encoder/layer_5/attention/output/dense/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/attention/output/dense/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_5/attention/output/dense/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_501/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_501" op: "Mul" input: "Mul_501/x" input: "bert/encoder/layer_5/attention/output/dense/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_502/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_502" op: "Mul" input: "Mul_502/x" input: "clip_by_global_norm/clip_by_global_norm/_92" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_316" op: "Add" input: "Mul_501" input: "Mul_502" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_503/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_503" op: "Mul" input: "Mul_503/x" input: "bert/encoder/layer_5/attention/output/dense/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_92" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_92" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_504/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_504" op: "Mul" input: "Mul_504/x" input: "Square_92" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_317" op: "Add" input: "Mul_503" input: "Mul_504" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_92" op: "Sqrt" input: "add_317" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_318/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_318" op: "Add" input: "Sqrt_92" input: "add_318/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_93" op: "RealDiv" input: "add_316" input: "add_318" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_505" op: "Mul" input: "add_2" input: "truediv_93" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_93" op: "Sub" input: "bert/encoder/layer_5/attention/output/dense/bias/read" input: "mul_505" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_482" op: "Assign" input: "bert/encoder/layer_5/attention/output/dense/bias" input: "sub_93" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_483" op: "Assign" input: "bert/encoder/layer_5/attention/output/dense/bias/adam_m" input: "add_316" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_484" op: "Assign" input: "bert/encoder/layer_5/attention/output/dense/bias/adam_v" input: "add_317" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_m/read" op: "Identity" input: "bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_v/read" op: "Identity" input: "bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_506/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_506" op: "Mul" input: "Mul_506/x" input: "bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_507/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_507" op: "Mul" input: "Mul_507/x" input: "clip_by_global_norm/clip_by_global_norm/_93" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_319" op: "Add" input: "Mul_506" input: "Mul_507" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_508/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_508" op: "Mul" input: "Mul_508/x" input: "bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_93" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_93" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_509/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_509" op: "Mul" input: "Mul_509/x" input: "Square_93" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_320" op: "Add" input: "Mul_508" input: "Mul_509" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_93" op: "Sqrt" input: "add_320" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_321/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_321" op: "Add" input: "Sqrt_93" input: "add_321/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_94" op: "RealDiv" input: "add_319" input: "add_321" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_510" op: "Mul" input: "add_2" input: "truediv_94" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_94" op: "Sub" input: "bert/encoder/layer_5/attention/output/LayerNorm/beta/read" input: "mul_510" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_485" op: "Assign" input: "bert/encoder/layer_5/attention/output/LayerNorm/beta" input: "sub_94" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_486" op: "Assign" input: "bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_m" input: "add_319" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_487" op: "Assign" input: "bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_v" input: "add_320" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m/read" op: "Identity" input: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v/read" op: "Identity" input: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_511/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_511" op: "Mul" input: "Mul_511/x" input: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_512/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_512" op: "Mul" input: "Mul_512/x" input: "clip_by_global_norm/clip_by_global_norm/_94" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_322" op: "Add" input: "Mul_511" input: "Mul_512" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_513/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_513" op: "Mul" input: "Mul_513/x" input: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_94" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_94" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_514/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_514" op: "Mul" input: "Mul_514/x" input: "Square_94" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_323" op: "Add" input: "Mul_513" input: "Mul_514" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_94" op: "Sqrt" input: "add_323" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_324/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_324" op: "Add" input: "Sqrt_94" input: "add_324/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_95" op: "RealDiv" input: "add_322" input: "add_324" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_515" op: "Mul" input: "add_2" input: "truediv_95" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_95" op: "Sub" input: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/read" input: "mul_515" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_488" op: "Assign" input: "bert/encoder/layer_5/attention/output/LayerNorm/gamma" input: "sub_95" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_489" op: "Assign" input: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m" input: "add_322" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_490" op: "Assign" input: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v" input: "add_323" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_516/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_516" op: "Mul" input: "Mul_516/x" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_517/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_517" op: "Mul" input: "Mul_517/x" input: "clip_by_global_norm/clip_by_global_norm/_95" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_325" op: "Add" input: "Mul_516" input: "Mul_517" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_518/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_518" op: "Mul" input: "Mul_518/x" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Square_95" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_95" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_519/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_519" op: "Mul" input: "Mul_519/x" input: "Square_95" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_326" op: "Add" input: "Mul_518" input: "Mul_519" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Sqrt_95" op: "Sqrt" input: "add_326" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_327/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_327" op: "Add" input: "Sqrt_95" input: "add_327/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "truediv_96" op: "RealDiv" input: "add_325" input: "add_327" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "mul_520/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_520" op: "Mul" input: "mul_520/x" input: "bert/encoder/layer_5/intermediate/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_328" op: "Add" input: "truediv_96" input: "mul_520" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "mul_521" op: "Mul" input: "add_2" input: "add_328" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "sub_96" op: "Sub" input: "bert/encoder/layer_5/intermediate/dense/kernel/read" input: "mul_521" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Assign_491" op: "Assign" input: "bert/encoder/layer_5/intermediate/dense/kernel" input: "sub_96" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_492" op: "Assign" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m" input: "add_325" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_493" op: "Assign" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v" input: "add_326" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_5/intermediate/dense/bias/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/intermediate/dense/bias/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_5/intermediate/dense/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/intermediate/dense/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_m" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/intermediate/dense/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_5/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_5/intermediate/dense/bias/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/intermediate/dense/bias/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_5/intermediate/dense/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/intermediate/dense/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_v" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/intermediate/dense/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_522/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_522" op: "Mul" input: "Mul_522/x" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_523/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_523" op: "Mul" input: "Mul_523/x" input: "clip_by_global_norm/clip_by_global_norm/_96" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_329" op: "Add" input: "Mul_522" input: "Mul_523" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_524/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_524" op: "Mul" input: "Mul_524/x" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Square_96" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_96" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_525/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_525" op: "Mul" input: "Mul_525/x" input: "Square_96" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_330" op: "Add" input: "Mul_524" input: "Mul_525" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Sqrt_96" op: "Sqrt" input: "add_330" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_331/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_331" op: "Add" input: "Sqrt_96" input: "add_331/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "truediv_97" op: "RealDiv" input: "add_329" input: "add_331" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "mul_526" op: "Mul" input: "add_2" input: "truediv_97" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "sub_97" op: "Sub" input: "bert/encoder/layer_5/intermediate/dense/bias/read" input: "mul_526" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Assign_494" op: "Assign" input: "bert/encoder/layer_5/intermediate/dense/bias" input: "sub_97" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_495" op: "Assign" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_m" input: "add_329" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_496" op: "Assign" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_v" input: "add_330" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_5/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_5/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_5/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_5/output/dense/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/output/dense/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_5/output/dense/kernel/adam_m" input: "bert/encoder/layer_5/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/output/dense/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_5/output/dense/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_5/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_5/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_5/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_5/output/dense/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/output/dense/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_5/output/dense/kernel/adam_v" input: "bert/encoder/layer_5/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/output/dense/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_5/output/dense/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_527/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_527" op: "Mul" input: "Mul_527/x" input: "bert/encoder/layer_5/output/dense/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_528/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_528" op: "Mul" input: "Mul_528/x" input: "clip_by_global_norm/clip_by_global_norm/_97" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_332" op: "Add" input: "Mul_527" input: "Mul_528" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_529/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_529" op: "Mul" input: "Mul_529/x" input: "bert/encoder/layer_5/output/dense/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Square_97" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_97" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_530/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_530" op: "Mul" input: "Mul_530/x" input: "Square_97" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_333" op: "Add" input: "Mul_529" input: "Mul_530" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Sqrt_97" op: "Sqrt" input: "add_333" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_334/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_334" op: "Add" input: "Sqrt_97" input: "add_334/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "truediv_98" op: "RealDiv" input: "add_332" input: "add_334" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "mul_531/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_531" op: "Mul" input: "mul_531/x" input: "bert/encoder/layer_5/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_335" op: "Add" input: "truediv_98" input: "mul_531" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "mul_532" op: "Mul" input: "add_2" input: "add_335" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "sub_98" op: "Sub" input: "bert/encoder/layer_5/output/dense/kernel/read" input: "mul_532" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Assign_497" op: "Assign" input: "bert/encoder/layer_5/output/dense/kernel" input: "sub_98" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_498" op: "Assign" input: "bert/encoder/layer_5/output/dense/kernel/adam_m" input: "add_332" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_499" op: "Assign" input: "bert/encoder/layer_5/output/dense/kernel/adam_v" input: "add_333" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/output/dense/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/output/dense/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_5/output/dense/bias/adam_m" input: "bert/encoder/layer_5/output/dense/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/output/dense/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_5/output/dense/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/output/dense/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/output/dense/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_5/output/dense/bias/adam_v" input: "bert/encoder/layer_5/output/dense/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/output/dense/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_5/output/dense/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_533/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_533" op: "Mul" input: "Mul_533/x" input: "bert/encoder/layer_5/output/dense/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_534/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_534" op: "Mul" input: "Mul_534/x" input: "clip_by_global_norm/clip_by_global_norm/_98" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_336" op: "Add" input: "Mul_533" input: "Mul_534" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_535/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_535" op: "Mul" input: "Mul_535/x" input: "bert/encoder/layer_5/output/dense/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_98" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_98" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_536/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_536" op: "Mul" input: "Mul_536/x" input: "Square_98" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_337" op: "Add" input: "Mul_535" input: "Mul_536" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_98" op: "Sqrt" input: "add_337" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_338/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_338" op: "Add" input: "Sqrt_98" input: "add_338/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_99" op: "RealDiv" input: "add_336" input: "add_338" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_537" op: "Mul" input: "add_2" input: "truediv_99" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_99" op: "Sub" input: "bert/encoder/layer_5/output/dense/bias/read" input: "mul_537" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_500" op: "Assign" input: "bert/encoder/layer_5/output/dense/bias" input: "sub_99" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_501" op: "Assign" input: "bert/encoder/layer_5/output/dense/bias/adam_m" input: "add_336" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_502" op: "Assign" input: "bert/encoder/layer_5/output/dense/bias/adam_v" input: "add_337" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/output/LayerNorm/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/output/LayerNorm/beta/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/output/LayerNorm/beta/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_5/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_5/output/LayerNorm/beta/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/output/LayerNorm/beta/adam_m/read" op: "Identity" input: "bert/encoder/layer_5/output/LayerNorm/beta/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/output/LayerNorm/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/output/LayerNorm/beta/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/output/LayerNorm/beta/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_5/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_5/output/LayerNorm/beta/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/output/LayerNorm/beta/adam_v/read" op: "Identity" input: "bert/encoder/layer_5/output/LayerNorm/beta/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_538/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_538" op: "Mul" input: "Mul_538/x" input: "bert/encoder/layer_5/output/LayerNorm/beta/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_539/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_539" op: "Mul" input: "Mul_539/x" input: "clip_by_global_norm/clip_by_global_norm/_99" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_339" op: "Add" input: "Mul_538" input: "Mul_539" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_540/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_540" op: "Mul" input: "Mul_540/x" input: "bert/encoder/layer_5/output/LayerNorm/beta/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_99" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_99" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_541/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_541" op: "Mul" input: "Mul_541/x" input: "Square_99" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_340" op: "Add" input: "Mul_540" input: "Mul_541" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_99" op: "Sqrt" input: "add_340" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_341/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_341" op: "Add" input: "Sqrt_99" input: "add_341/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_100" op: "RealDiv" input: "add_339" input: "add_341" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_542" op: "Mul" input: "add_2" input: "truediv_100" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_100" op: "Sub" input: "bert/encoder/layer_5/output/LayerNorm/beta/read" input: "mul_542" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_503" op: "Assign" input: "bert/encoder/layer_5/output/LayerNorm/beta" input: "sub_100" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_504" op: "Assign" input: "bert/encoder/layer_5/output/LayerNorm/beta/adam_m" input: "add_339" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_505" op: "Assign" input: "bert/encoder/layer_5/output/LayerNorm/beta/adam_v" input: "add_340" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/output/LayerNorm/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/output/LayerNorm/gamma/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/output/LayerNorm/gamma/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_5/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_5/output/LayerNorm/gamma/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/output/LayerNorm/gamma/adam_m/read" op: "Identity" input: "bert/encoder/layer_5/output/LayerNorm/gamma/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/output/LayerNorm/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/output/LayerNorm/gamma/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_5/output/LayerNorm/gamma/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_5/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_5/output/LayerNorm/gamma/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_5/output/LayerNorm/gamma/adam_v/read" op: "Identity" input: "bert/encoder/layer_5/output/LayerNorm/gamma/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_543/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_543" op: "Mul" input: "Mul_543/x" input: "bert/encoder/layer_5/output/LayerNorm/gamma/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_544/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_544" op: "Mul" input: "Mul_544/x" input: "clip_by_global_norm/clip_by_global_norm/_100" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_342" op: "Add" input: "Mul_543" input: "Mul_544" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_545/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_545" op: "Mul" input: "Mul_545/x" input: "bert/encoder/layer_5/output/LayerNorm/gamma/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_100" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_100" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_546/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_546" op: "Mul" input: "Mul_546/x" input: "Square_100" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_343" op: "Add" input: "Mul_545" input: "Mul_546" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_100" op: "Sqrt" input: "add_343" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_344/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_344" op: "Add" input: "Sqrt_100" input: "add_344/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_101" op: "RealDiv" input: "add_342" input: "add_344" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_547" op: "Mul" input: "add_2" input: "truediv_101" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_101" op: "Sub" input: "bert/encoder/layer_5/output/LayerNorm/gamma/read" input: "mul_547" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_506" op: "Assign" input: "bert/encoder/layer_5/output/LayerNorm/gamma" input: "sub_101" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_507" op: "Assign" input: "bert/encoder/layer_5/output/LayerNorm/gamma/adam_m" input: "add_342" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_508" op: "Assign" input: "bert/encoder/layer_5/output/LayerNorm/gamma/adam_v" input: "add_343" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_m" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_v" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_548/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_548" op: "Mul" input: "Mul_548/x" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_549/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_549" op: "Mul" input: "Mul_549/x" input: "clip_by_global_norm/clip_by_global_norm/_101" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_345" op: "Add" input: "Mul_548" input: "Mul_549" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_550/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_550" op: "Mul" input: "Mul_550/x" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_101" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_101" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_551/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_551" op: "Mul" input: "Mul_551/x" input: "Square_101" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_346" op: "Add" input: "Mul_550" input: "Mul_551" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_101" op: "Sqrt" input: "add_346" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_347/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_347" op: "Add" input: "Sqrt_101" input: "add_347/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_102" op: "RealDiv" input: "add_345" input: "add_347" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_552/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_552" op: "Mul" input: "mul_552/x" input: "bert/encoder/layer_6/attention/self/query/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_348" op: "Add" input: "truediv_102" input: "mul_552" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_553" op: "Mul" input: "add_2" input: "add_348" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_102" op: "Sub" input: "bert/encoder/layer_6/attention/self/query/kernel/read" input: "mul_553" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_509" op: "Assign" input: "bert/encoder/layer_6/attention/self/query/kernel" input: "sub_102" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_510" op: "Assign" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_m" input: "add_345" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_511" op: "Assign" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_v" input: "add_346" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/attention/self/query/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/self/query/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/attention/self/query/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_6/attention/self/query/bias/adam_m" input: "bert/encoder/layer_6/attention/self/query/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/attention/self/query/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_6/attention/self/query/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/self/query/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/self/query/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/attention/self/query/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_6/attention/self/query/bias/adam_v" input: "bert/encoder/layer_6/attention/self/query/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/attention/self/query/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_6/attention/self/query/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_554/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_554" op: "Mul" input: "Mul_554/x" input: "bert/encoder/layer_6/attention/self/query/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_555/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_555" op: "Mul" input: "Mul_555/x" input: "clip_by_global_norm/clip_by_global_norm/_102" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_349" op: "Add" input: "Mul_554" input: "Mul_555" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_556/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_556" op: "Mul" input: "Mul_556/x" input: "bert/encoder/layer_6/attention/self/query/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_102" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_102" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_557/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_557" op: "Mul" input: "Mul_557/x" input: "Square_102" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_350" op: "Add" input: "Mul_556" input: "Mul_557" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_102" op: "Sqrt" input: "add_350" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_351/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_351" op: "Add" input: "Sqrt_102" input: "add_351/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_103" op: "RealDiv" input: "add_349" input: "add_351" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_558" op: "Mul" input: "add_2" input: "truediv_103" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_103" op: "Sub" input: "bert/encoder/layer_6/attention/self/query/bias/read" input: "mul_558" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_512" op: "Assign" input: "bert/encoder/layer_6/attention/self/query/bias" input: "sub_103" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_513" op: "Assign" input: "bert/encoder/layer_6/attention/self/query/bias/adam_m" input: "add_349" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_514" op: "Assign" input: "bert/encoder/layer_6/attention/self/query/bias/adam_v" input: "add_350" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_m" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_v" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_559/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_559" op: "Mul" input: "Mul_559/x" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_560/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_560" op: "Mul" input: "Mul_560/x" input: "clip_by_global_norm/clip_by_global_norm/_103" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_352" op: "Add" input: "Mul_559" input: "Mul_560" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_561/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_561" op: "Mul" input: "Mul_561/x" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_103" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_103" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_562/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_562" op: "Mul" input: "Mul_562/x" input: "Square_103" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_353" op: "Add" input: "Mul_561" input: "Mul_562" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_103" op: "Sqrt" input: "add_353" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_354/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_354" op: "Add" input: "Sqrt_103" input: "add_354/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_104" op: "RealDiv" input: "add_352" input: "add_354" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_563/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_563" op: "Mul" input: "mul_563/x" input: "bert/encoder/layer_6/attention/self/key/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_355" op: "Add" input: "truediv_104" input: "mul_563" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_564" op: "Mul" input: "add_2" input: "add_355" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_104" op: "Sub" input: "bert/encoder/layer_6/attention/self/key/kernel/read" input: "mul_564" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_515" op: "Assign" input: "bert/encoder/layer_6/attention/self/key/kernel" input: "sub_104" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_516" op: "Assign" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_m" input: "add_352" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_517" op: "Assign" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_v" input: "add_353" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/attention/self/key/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/self/key/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/attention/self/key/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_6/attention/self/key/bias/adam_m" input: "bert/encoder/layer_6/attention/self/key/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/attention/self/key/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_6/attention/self/key/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/self/key/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/self/key/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/attention/self/key/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_6/attention/self/key/bias/adam_v" input: "bert/encoder/layer_6/attention/self/key/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/attention/self/key/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_6/attention/self/key/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_565/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_565" op: "Mul" input: "Mul_565/x" input: "bert/encoder/layer_6/attention/self/key/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_566/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_566" op: "Mul" input: "Mul_566/x" input: "clip_by_global_norm/clip_by_global_norm/_104" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_356" op: "Add" input: "Mul_565" input: "Mul_566" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_567/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_567" op: "Mul" input: "Mul_567/x" input: "bert/encoder/layer_6/attention/self/key/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_104" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_104" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_568/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_568" op: "Mul" input: "Mul_568/x" input: "Square_104" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_357" op: "Add" input: "Mul_567" input: "Mul_568" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_104" op: "Sqrt" input: "add_357" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_358/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_358" op: "Add" input: "Sqrt_104" input: "add_358/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_105" op: "RealDiv" input: "add_356" input: "add_358" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_569" op: "Mul" input: "add_2" input: "truediv_105" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_105" op: "Sub" input: "bert/encoder/layer_6/attention/self/key/bias/read" input: "mul_569" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_518" op: "Assign" input: "bert/encoder/layer_6/attention/self/key/bias" input: "sub_105" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_519" op: "Assign" input: "bert/encoder/layer_6/attention/self/key/bias/adam_m" input: "add_356" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_520" op: "Assign" input: "bert/encoder/layer_6/attention/self/key/bias/adam_v" input: "add_357" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_m" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_v" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_570/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_570" op: "Mul" input: "Mul_570/x" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_571/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_571" op: "Mul" input: "Mul_571/x" input: "clip_by_global_norm/clip_by_global_norm/_105" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_359" op: "Add" input: "Mul_570" input: "Mul_571" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_572/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_572" op: "Mul" input: "Mul_572/x" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_105" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_105" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_573/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_573" op: "Mul" input: "Mul_573/x" input: "Square_105" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_360" op: "Add" input: "Mul_572" input: "Mul_573" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_105" op: "Sqrt" input: "add_360" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_361/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_361" op: "Add" input: "Sqrt_105" input: "add_361/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_106" op: "RealDiv" input: "add_359" input: "add_361" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_574/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_574" op: "Mul" input: "mul_574/x" input: "bert/encoder/layer_6/attention/self/value/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_362" op: "Add" input: "truediv_106" input: "mul_574" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_575" op: "Mul" input: "add_2" input: "add_362" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_106" op: "Sub" input: "bert/encoder/layer_6/attention/self/value/kernel/read" input: "mul_575" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_521" op: "Assign" input: "bert/encoder/layer_6/attention/self/value/kernel" input: "sub_106" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_522" op: "Assign" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_m" input: "add_359" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_523" op: "Assign" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_v" input: "add_360" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/attention/self/value/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/self/value/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/attention/self/value/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_6/attention/self/value/bias/adam_m" input: "bert/encoder/layer_6/attention/self/value/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/attention/self/value/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_6/attention/self/value/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/self/value/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/self/value/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/attention/self/value/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_6/attention/self/value/bias/adam_v" input: "bert/encoder/layer_6/attention/self/value/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/attention/self/value/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_6/attention/self/value/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_576/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_576" op: "Mul" input: "Mul_576/x" input: "bert/encoder/layer_6/attention/self/value/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_577/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_577" op: "Mul" input: "Mul_577/x" input: "clip_by_global_norm/clip_by_global_norm/_106" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_363" op: "Add" input: "Mul_576" input: "Mul_577" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_578/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_578" op: "Mul" input: "Mul_578/x" input: "bert/encoder/layer_6/attention/self/value/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_106" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_106" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_579/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_579" op: "Mul" input: "Mul_579/x" input: "Square_106" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_364" op: "Add" input: "Mul_578" input: "Mul_579" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_106" op: "Sqrt" input: "add_364" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_365/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_365" op: "Add" input: "Sqrt_106" input: "add_365/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_107" op: "RealDiv" input: "add_363" input: "add_365" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_580" op: "Mul" input: "add_2" input: "truediv_107" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_107" op: "Sub" input: "bert/encoder/layer_6/attention/self/value/bias/read" input: "mul_580" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_524" op: "Assign" input: "bert/encoder/layer_6/attention/self/value/bias" input: "sub_107" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_525" op: "Assign" input: "bert/encoder/layer_6/attention/self/value/bias/adam_m" input: "add_363" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_526" op: "Assign" input: "bert/encoder/layer_6/attention/self/value/bias/adam_v" input: "add_364" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_581/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_581" op: "Mul" input: "Mul_581/x" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_582/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_582" op: "Mul" input: "Mul_582/x" input: "clip_by_global_norm/clip_by_global_norm/_107" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_366" op: "Add" input: "Mul_581" input: "Mul_582" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_583/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_583" op: "Mul" input: "Mul_583/x" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_107" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_107" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_584/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_584" op: "Mul" input: "Mul_584/x" input: "Square_107" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_367" op: "Add" input: "Mul_583" input: "Mul_584" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_107" op: "Sqrt" input: "add_367" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_368/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_368" op: "Add" input: "Sqrt_107" input: "add_368/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_108" op: "RealDiv" input: "add_366" input: "add_368" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_585/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_585" op: "Mul" input: "mul_585/x" input: "bert/encoder/layer_6/attention/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_369" op: "Add" input: "truediv_108" input: "mul_585" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_586" op: "Mul" input: "add_2" input: "add_369" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_108" op: "Sub" input: "bert/encoder/layer_6/attention/output/dense/kernel/read" input: "mul_586" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_527" op: "Assign" input: "bert/encoder/layer_6/attention/output/dense/kernel" input: "sub_108" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_528" op: "Assign" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m" input: "add_366" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_529" op: "Assign" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v" input: "add_367" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/attention/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/output/dense/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/attention/output/dense/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_6/attention/output/dense/bias/adam_m" input: "bert/encoder/layer_6/attention/output/dense/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/attention/output/dense/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_6/attention/output/dense/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/output/dense/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/attention/output/dense/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_6/attention/output/dense/bias/adam_v" input: "bert/encoder/layer_6/attention/output/dense/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/attention/output/dense/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_6/attention/output/dense/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_587/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_587" op: "Mul" input: "Mul_587/x" input: "bert/encoder/layer_6/attention/output/dense/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_588/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_588" op: "Mul" input: "Mul_588/x" input: "clip_by_global_norm/clip_by_global_norm/_108" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_370" op: "Add" input: "Mul_587" input: "Mul_588" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_589/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_589" op: "Mul" input: "Mul_589/x" input: "bert/encoder/layer_6/attention/output/dense/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_108" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_108" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_590/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_590" op: "Mul" input: "Mul_590/x" input: "Square_108" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_371" op: "Add" input: "Mul_589" input: "Mul_590" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_108" op: "Sqrt" input: "add_371" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_372/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_372" op: "Add" input: "Sqrt_108" input: "add_372/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_109" op: "RealDiv" input: "add_370" input: "add_372" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_591" op: "Mul" input: "add_2" input: "truediv_109" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_109" op: "Sub" input: "bert/encoder/layer_6/attention/output/dense/bias/read" input: "mul_591" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_530" op: "Assign" input: "bert/encoder/layer_6/attention/output/dense/bias" input: "sub_109" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_531" op: "Assign" input: "bert/encoder/layer_6/attention/output/dense/bias/adam_m" input: "add_370" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_532" op: "Assign" input: "bert/encoder/layer_6/attention/output/dense/bias/adam_v" input: "add_371" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_m/read" op: "Identity" input: "bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_v/read" op: "Identity" input: "bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_592/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_592" op: "Mul" input: "Mul_592/x" input: "bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_593/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_593" op: "Mul" input: "Mul_593/x" input: "clip_by_global_norm/clip_by_global_norm/_109" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_373" op: "Add" input: "Mul_592" input: "Mul_593" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_594/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_594" op: "Mul" input: "Mul_594/x" input: "bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_109" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_109" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_595/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_595" op: "Mul" input: "Mul_595/x" input: "Square_109" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_374" op: "Add" input: "Mul_594" input: "Mul_595" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_109" op: "Sqrt" input: "add_374" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_375/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_375" op: "Add" input: "Sqrt_109" input: "add_375/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_110" op: "RealDiv" input: "add_373" input: "add_375" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_596" op: "Mul" input: "add_2" input: "truediv_110" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_110" op: "Sub" input: "bert/encoder/layer_6/attention/output/LayerNorm/beta/read" input: "mul_596" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_533" op: "Assign" input: "bert/encoder/layer_6/attention/output/LayerNorm/beta" input: "sub_110" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_534" op: "Assign" input: "bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_m" input: "add_373" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_535" op: "Assign" input: "bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_v" input: "add_374" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m/read" op: "Identity" input: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v/read" op: "Identity" input: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_597/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_597" op: "Mul" input: "Mul_597/x" input: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_598/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_598" op: "Mul" input: "Mul_598/x" input: "clip_by_global_norm/clip_by_global_norm/_110" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_376" op: "Add" input: "Mul_597" input: "Mul_598" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_599/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_599" op: "Mul" input: "Mul_599/x" input: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_110" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_110" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_600/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_600" op: "Mul" input: "Mul_600/x" input: "Square_110" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_377" op: "Add" input: "Mul_599" input: "Mul_600" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_110" op: "Sqrt" input: "add_377" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_378/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_378" op: "Add" input: "Sqrt_110" input: "add_378/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_111" op: "RealDiv" input: "add_376" input: "add_378" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_601" op: "Mul" input: "add_2" input: "truediv_111" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_111" op: "Sub" input: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/read" input: "mul_601" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_536" op: "Assign" input: "bert/encoder/layer_6/attention/output/LayerNorm/gamma" input: "sub_111" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_537" op: "Assign" input: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m" input: "add_376" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_538" op: "Assign" input: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v" input: "add_377" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_602/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_602" op: "Mul" input: "Mul_602/x" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_603/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_603" op: "Mul" input: "Mul_603/x" input: "clip_by_global_norm/clip_by_global_norm/_111" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_379" op: "Add" input: "Mul_602" input: "Mul_603" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_604/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_604" op: "Mul" input: "Mul_604/x" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Square_111" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_111" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_605/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_605" op: "Mul" input: "Mul_605/x" input: "Square_111" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_380" op: "Add" input: "Mul_604" input: "Mul_605" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Sqrt_111" op: "Sqrt" input: "add_380" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_381/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_381" op: "Add" input: "Sqrt_111" input: "add_381/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "truediv_112" op: "RealDiv" input: "add_379" input: "add_381" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "mul_606/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_606" op: "Mul" input: "mul_606/x" input: "bert/encoder/layer_6/intermediate/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_382" op: "Add" input: "truediv_112" input: "mul_606" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "mul_607" op: "Mul" input: "add_2" input: "add_382" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "sub_112" op: "Sub" input: "bert/encoder/layer_6/intermediate/dense/kernel/read" input: "mul_607" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Assign_539" op: "Assign" input: "bert/encoder/layer_6/intermediate/dense/kernel" input: "sub_112" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_540" op: "Assign" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m" input: "add_379" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_541" op: "Assign" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v" input: "add_380" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_6/intermediate/dense/bias/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/intermediate/dense/bias/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_6/intermediate/dense/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/intermediate/dense/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_m" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/intermediate/dense/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_6/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_6/intermediate/dense/bias/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/intermediate/dense/bias/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_6/intermediate/dense/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/intermediate/dense/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_v" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/intermediate/dense/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_608/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_608" op: "Mul" input: "Mul_608/x" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_609/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_609" op: "Mul" input: "Mul_609/x" input: "clip_by_global_norm/clip_by_global_norm/_112" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_383" op: "Add" input: "Mul_608" input: "Mul_609" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_610/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_610" op: "Mul" input: "Mul_610/x" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Square_112" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_112" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_611/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_611" op: "Mul" input: "Mul_611/x" input: "Square_112" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_384" op: "Add" input: "Mul_610" input: "Mul_611" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Sqrt_112" op: "Sqrt" input: "add_384" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_385/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_385" op: "Add" input: "Sqrt_112" input: "add_385/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "truediv_113" op: "RealDiv" input: "add_383" input: "add_385" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "mul_612" op: "Mul" input: "add_2" input: "truediv_113" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "sub_113" op: "Sub" input: "bert/encoder/layer_6/intermediate/dense/bias/read" input: "mul_612" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Assign_542" op: "Assign" input: "bert/encoder/layer_6/intermediate/dense/bias" input: "sub_113" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_543" op: "Assign" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_m" input: "add_383" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_544" op: "Assign" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_v" input: "add_384" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_6/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_6/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_6/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_6/output/dense/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/output/dense/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_6/output/dense/kernel/adam_m" input: "bert/encoder/layer_6/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/output/dense/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_6/output/dense/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_6/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_6/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_6/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_6/output/dense/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/output/dense/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_6/output/dense/kernel/adam_v" input: "bert/encoder/layer_6/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/output/dense/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_6/output/dense/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_613/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_613" op: "Mul" input: "Mul_613/x" input: "bert/encoder/layer_6/output/dense/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_614/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_614" op: "Mul" input: "Mul_614/x" input: "clip_by_global_norm/clip_by_global_norm/_113" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_386" op: "Add" input: "Mul_613" input: "Mul_614" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_615/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_615" op: "Mul" input: "Mul_615/x" input: "bert/encoder/layer_6/output/dense/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Square_113" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_113" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_616/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_616" op: "Mul" input: "Mul_616/x" input: "Square_113" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_387" op: "Add" input: "Mul_615" input: "Mul_616" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Sqrt_113" op: "Sqrt" input: "add_387" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_388/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_388" op: "Add" input: "Sqrt_113" input: "add_388/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "truediv_114" op: "RealDiv" input: "add_386" input: "add_388" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "mul_617/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_617" op: "Mul" input: "mul_617/x" input: "bert/encoder/layer_6/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_389" op: "Add" input: "truediv_114" input: "mul_617" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "mul_618" op: "Mul" input: "add_2" input: "add_389" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "sub_114" op: "Sub" input: "bert/encoder/layer_6/output/dense/kernel/read" input: "mul_618" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Assign_545" op: "Assign" input: "bert/encoder/layer_6/output/dense/kernel" input: "sub_114" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_546" op: "Assign" input: "bert/encoder/layer_6/output/dense/kernel/adam_m" input: "add_386" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_547" op: "Assign" input: "bert/encoder/layer_6/output/dense/kernel/adam_v" input: "add_387" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/output/dense/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/output/dense/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_6/output/dense/bias/adam_m" input: "bert/encoder/layer_6/output/dense/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/output/dense/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_6/output/dense/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/output/dense/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/output/dense/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_6/output/dense/bias/adam_v" input: "bert/encoder/layer_6/output/dense/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/output/dense/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_6/output/dense/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_619/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_619" op: "Mul" input: "Mul_619/x" input: "bert/encoder/layer_6/output/dense/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_620/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_620" op: "Mul" input: "Mul_620/x" input: "clip_by_global_norm/clip_by_global_norm/_114" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_390" op: "Add" input: "Mul_619" input: "Mul_620" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_621/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_621" op: "Mul" input: "Mul_621/x" input: "bert/encoder/layer_6/output/dense/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_114" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_114" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_622/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_622" op: "Mul" input: "Mul_622/x" input: "Square_114" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_391" op: "Add" input: "Mul_621" input: "Mul_622" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_114" op: "Sqrt" input: "add_391" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_392/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_392" op: "Add" input: "Sqrt_114" input: "add_392/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_115" op: "RealDiv" input: "add_390" input: "add_392" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_623" op: "Mul" input: "add_2" input: "truediv_115" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_115" op: "Sub" input: "bert/encoder/layer_6/output/dense/bias/read" input: "mul_623" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_548" op: "Assign" input: "bert/encoder/layer_6/output/dense/bias" input: "sub_115" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_549" op: "Assign" input: "bert/encoder/layer_6/output/dense/bias/adam_m" input: "add_390" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_550" op: "Assign" input: "bert/encoder/layer_6/output/dense/bias/adam_v" input: "add_391" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/output/LayerNorm/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/output/LayerNorm/beta/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/output/LayerNorm/beta/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_6/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_6/output/LayerNorm/beta/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/output/LayerNorm/beta/adam_m/read" op: "Identity" input: "bert/encoder/layer_6/output/LayerNorm/beta/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/output/LayerNorm/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/output/LayerNorm/beta/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/output/LayerNorm/beta/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_6/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_6/output/LayerNorm/beta/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/output/LayerNorm/beta/adam_v/read" op: "Identity" input: "bert/encoder/layer_6/output/LayerNorm/beta/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_624/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_624" op: "Mul" input: "Mul_624/x" input: "bert/encoder/layer_6/output/LayerNorm/beta/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_625/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_625" op: "Mul" input: "Mul_625/x" input: "clip_by_global_norm/clip_by_global_norm/_115" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_393" op: "Add" input: "Mul_624" input: "Mul_625" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_626/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_626" op: "Mul" input: "Mul_626/x" input: "bert/encoder/layer_6/output/LayerNorm/beta/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_115" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_115" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_627/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_627" op: "Mul" input: "Mul_627/x" input: "Square_115" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_394" op: "Add" input: "Mul_626" input: "Mul_627" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_115" op: "Sqrt" input: "add_394" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_395/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_395" op: "Add" input: "Sqrt_115" input: "add_395/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_116" op: "RealDiv" input: "add_393" input: "add_395" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_628" op: "Mul" input: "add_2" input: "truediv_116" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_116" op: "Sub" input: "bert/encoder/layer_6/output/LayerNorm/beta/read" input: "mul_628" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_551" op: "Assign" input: "bert/encoder/layer_6/output/LayerNorm/beta" input: "sub_116" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_552" op: "Assign" input: "bert/encoder/layer_6/output/LayerNorm/beta/adam_m" input: "add_393" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_553" op: "Assign" input: "bert/encoder/layer_6/output/LayerNorm/beta/adam_v" input: "add_394" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/output/LayerNorm/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/output/LayerNorm/gamma/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/output/LayerNorm/gamma/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_6/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_6/output/LayerNorm/gamma/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/output/LayerNorm/gamma/adam_m/read" op: "Identity" input: "bert/encoder/layer_6/output/LayerNorm/gamma/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/output/LayerNorm/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/output/LayerNorm/gamma/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_6/output/LayerNorm/gamma/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_6/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_6/output/LayerNorm/gamma/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_6/output/LayerNorm/gamma/adam_v/read" op: "Identity" input: "bert/encoder/layer_6/output/LayerNorm/gamma/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_629/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_629" op: "Mul" input: "Mul_629/x" input: "bert/encoder/layer_6/output/LayerNorm/gamma/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_630/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_630" op: "Mul" input: "Mul_630/x" input: "clip_by_global_norm/clip_by_global_norm/_116" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_396" op: "Add" input: "Mul_629" input: "Mul_630" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_631/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_631" op: "Mul" input: "Mul_631/x" input: "bert/encoder/layer_6/output/LayerNorm/gamma/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_116" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_116" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_632/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_632" op: "Mul" input: "Mul_632/x" input: "Square_116" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_397" op: "Add" input: "Mul_631" input: "Mul_632" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_116" op: "Sqrt" input: "add_397" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_398/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_398" op: "Add" input: "Sqrt_116" input: "add_398/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_117" op: "RealDiv" input: "add_396" input: "add_398" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_633" op: "Mul" input: "add_2" input: "truediv_117" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_117" op: "Sub" input: "bert/encoder/layer_6/output/LayerNorm/gamma/read" input: "mul_633" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_554" op: "Assign" input: "bert/encoder/layer_6/output/LayerNorm/gamma" input: "sub_117" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_555" op: "Assign" input: "bert/encoder/layer_6/output/LayerNorm/gamma/adam_m" input: "add_396" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_556" op: "Assign" input: "bert/encoder/layer_6/output/LayerNorm/gamma/adam_v" input: "add_397" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_m" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_v" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_634/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_634" op: "Mul" input: "Mul_634/x" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_635/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_635" op: "Mul" input: "Mul_635/x" input: "clip_by_global_norm/clip_by_global_norm/_117" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_399" op: "Add" input: "Mul_634" input: "Mul_635" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_636/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_636" op: "Mul" input: "Mul_636/x" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_117" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_117" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_637/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_637" op: "Mul" input: "Mul_637/x" input: "Square_117" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_400" op: "Add" input: "Mul_636" input: "Mul_637" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_117" op: "Sqrt" input: "add_400" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_401/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_401" op: "Add" input: "Sqrt_117" input: "add_401/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_118" op: "RealDiv" input: "add_399" input: "add_401" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_638/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_638" op: "Mul" input: "mul_638/x" input: "bert/encoder/layer_7/attention/self/query/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_402" op: "Add" input: "truediv_118" input: "mul_638" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_639" op: "Mul" input: "add_2" input: "add_402" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_118" op: "Sub" input: "bert/encoder/layer_7/attention/self/query/kernel/read" input: "mul_639" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_557" op: "Assign" input: "bert/encoder/layer_7/attention/self/query/kernel" input: "sub_118" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_558" op: "Assign" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_m" input: "add_399" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_559" op: "Assign" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_v" input: "add_400" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/attention/self/query/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/self/query/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/attention/self/query/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_7/attention/self/query/bias/adam_m" input: "bert/encoder/layer_7/attention/self/query/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/attention/self/query/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_7/attention/self/query/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/self/query/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/self/query/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/attention/self/query/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_7/attention/self/query/bias/adam_v" input: "bert/encoder/layer_7/attention/self/query/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/attention/self/query/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_7/attention/self/query/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_640/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_640" op: "Mul" input: "Mul_640/x" input: "bert/encoder/layer_7/attention/self/query/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_641/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_641" op: "Mul" input: "Mul_641/x" input: "clip_by_global_norm/clip_by_global_norm/_118" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_403" op: "Add" input: "Mul_640" input: "Mul_641" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_642/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_642" op: "Mul" input: "Mul_642/x" input: "bert/encoder/layer_7/attention/self/query/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_118" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_118" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_643/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_643" op: "Mul" input: "Mul_643/x" input: "Square_118" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_404" op: "Add" input: "Mul_642" input: "Mul_643" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_118" op: "Sqrt" input: "add_404" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_405/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_405" op: "Add" input: "Sqrt_118" input: "add_405/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_119" op: "RealDiv" input: "add_403" input: "add_405" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_644" op: "Mul" input: "add_2" input: "truediv_119" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_119" op: "Sub" input: "bert/encoder/layer_7/attention/self/query/bias/read" input: "mul_644" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_560" op: "Assign" input: "bert/encoder/layer_7/attention/self/query/bias" input: "sub_119" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_561" op: "Assign" input: "bert/encoder/layer_7/attention/self/query/bias/adam_m" input: "add_403" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_562" op: "Assign" input: "bert/encoder/layer_7/attention/self/query/bias/adam_v" input: "add_404" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_m" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_v" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_645/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_645" op: "Mul" input: "Mul_645/x" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_646/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_646" op: "Mul" input: "Mul_646/x" input: "clip_by_global_norm/clip_by_global_norm/_119" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_406" op: "Add" input: "Mul_645" input: "Mul_646" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_647/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_647" op: "Mul" input: "Mul_647/x" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_119" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_119" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_648/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_648" op: "Mul" input: "Mul_648/x" input: "Square_119" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_407" op: "Add" input: "Mul_647" input: "Mul_648" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_119" op: "Sqrt" input: "add_407" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_408/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_408" op: "Add" input: "Sqrt_119" input: "add_408/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_120" op: "RealDiv" input: "add_406" input: "add_408" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_649/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_649" op: "Mul" input: "mul_649/x" input: "bert/encoder/layer_7/attention/self/key/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_409" op: "Add" input: "truediv_120" input: "mul_649" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_650" op: "Mul" input: "add_2" input: "add_409" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_120" op: "Sub" input: "bert/encoder/layer_7/attention/self/key/kernel/read" input: "mul_650" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_563" op: "Assign" input: "bert/encoder/layer_7/attention/self/key/kernel" input: "sub_120" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_564" op: "Assign" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_m" input: "add_406" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_565" op: "Assign" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_v" input: "add_407" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/attention/self/key/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/self/key/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/attention/self/key/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_7/attention/self/key/bias/adam_m" input: "bert/encoder/layer_7/attention/self/key/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/attention/self/key/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_7/attention/self/key/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/self/key/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/self/key/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/attention/self/key/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_7/attention/self/key/bias/adam_v" input: "bert/encoder/layer_7/attention/self/key/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/attention/self/key/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_7/attention/self/key/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_651/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_651" op: "Mul" input: "Mul_651/x" input: "bert/encoder/layer_7/attention/self/key/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_652/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_652" op: "Mul" input: "Mul_652/x" input: "clip_by_global_norm/clip_by_global_norm/_120" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_410" op: "Add" input: "Mul_651" input: "Mul_652" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_653/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_653" op: "Mul" input: "Mul_653/x" input: "bert/encoder/layer_7/attention/self/key/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_120" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_120" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_654/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_654" op: "Mul" input: "Mul_654/x" input: "Square_120" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_411" op: "Add" input: "Mul_653" input: "Mul_654" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_120" op: "Sqrt" input: "add_411" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_412/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_412" op: "Add" input: "Sqrt_120" input: "add_412/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_121" op: "RealDiv" input: "add_410" input: "add_412" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_655" op: "Mul" input: "add_2" input: "truediv_121" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_121" op: "Sub" input: "bert/encoder/layer_7/attention/self/key/bias/read" input: "mul_655" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_566" op: "Assign" input: "bert/encoder/layer_7/attention/self/key/bias" input: "sub_121" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_567" op: "Assign" input: "bert/encoder/layer_7/attention/self/key/bias/adam_m" input: "add_410" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_568" op: "Assign" input: "bert/encoder/layer_7/attention/self/key/bias/adam_v" input: "add_411" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_m" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_v" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_656/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_656" op: "Mul" input: "Mul_656/x" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_657/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_657" op: "Mul" input: "Mul_657/x" input: "clip_by_global_norm/clip_by_global_norm/_121" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_413" op: "Add" input: "Mul_656" input: "Mul_657" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_658/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_658" op: "Mul" input: "Mul_658/x" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_121" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_121" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_659/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_659" op: "Mul" input: "Mul_659/x" input: "Square_121" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_414" op: "Add" input: "Mul_658" input: "Mul_659" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_121" op: "Sqrt" input: "add_414" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_415/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_415" op: "Add" input: "Sqrt_121" input: "add_415/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_122" op: "RealDiv" input: "add_413" input: "add_415" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_660/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_660" op: "Mul" input: "mul_660/x" input: "bert/encoder/layer_7/attention/self/value/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_416" op: "Add" input: "truediv_122" input: "mul_660" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_661" op: "Mul" input: "add_2" input: "add_416" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_122" op: "Sub" input: "bert/encoder/layer_7/attention/self/value/kernel/read" input: "mul_661" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_569" op: "Assign" input: "bert/encoder/layer_7/attention/self/value/kernel" input: "sub_122" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_570" op: "Assign" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_m" input: "add_413" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_571" op: "Assign" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_v" input: "add_414" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/attention/self/value/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/self/value/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/attention/self/value/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_7/attention/self/value/bias/adam_m" input: "bert/encoder/layer_7/attention/self/value/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/attention/self/value/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_7/attention/self/value/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/self/value/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/self/value/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/attention/self/value/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_7/attention/self/value/bias/adam_v" input: "bert/encoder/layer_7/attention/self/value/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/attention/self/value/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_7/attention/self/value/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_662/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_662" op: "Mul" input: "Mul_662/x" input: "bert/encoder/layer_7/attention/self/value/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_663/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_663" op: "Mul" input: "Mul_663/x" input: "clip_by_global_norm/clip_by_global_norm/_122" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_417" op: "Add" input: "Mul_662" input: "Mul_663" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_664/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_664" op: "Mul" input: "Mul_664/x" input: "bert/encoder/layer_7/attention/self/value/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_122" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_122" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_665/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_665" op: "Mul" input: "Mul_665/x" input: "Square_122" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_418" op: "Add" input: "Mul_664" input: "Mul_665" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_122" op: "Sqrt" input: "add_418" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_419/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_419" op: "Add" input: "Sqrt_122" input: "add_419/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_123" op: "RealDiv" input: "add_417" input: "add_419" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_666" op: "Mul" input: "add_2" input: "truediv_123" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_123" op: "Sub" input: "bert/encoder/layer_7/attention/self/value/bias/read" input: "mul_666" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_572" op: "Assign" input: "bert/encoder/layer_7/attention/self/value/bias" input: "sub_123" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_573" op: "Assign" input: "bert/encoder/layer_7/attention/self/value/bias/adam_m" input: "add_417" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_574" op: "Assign" input: "bert/encoder/layer_7/attention/self/value/bias/adam_v" input: "add_418" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_667/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_667" op: "Mul" input: "Mul_667/x" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_668/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_668" op: "Mul" input: "Mul_668/x" input: "clip_by_global_norm/clip_by_global_norm/_123" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_420" op: "Add" input: "Mul_667" input: "Mul_668" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_669/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_669" op: "Mul" input: "Mul_669/x" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_123" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_123" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_670/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_670" op: "Mul" input: "Mul_670/x" input: "Square_123" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_421" op: "Add" input: "Mul_669" input: "Mul_670" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_123" op: "Sqrt" input: "add_421" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_422/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_422" op: "Add" input: "Sqrt_123" input: "add_422/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_124" op: "RealDiv" input: "add_420" input: "add_422" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_671/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_671" op: "Mul" input: "mul_671/x" input: "bert/encoder/layer_7/attention/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_423" op: "Add" input: "truediv_124" input: "mul_671" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_672" op: "Mul" input: "add_2" input: "add_423" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_124" op: "Sub" input: "bert/encoder/layer_7/attention/output/dense/kernel/read" input: "mul_672" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_575" op: "Assign" input: "bert/encoder/layer_7/attention/output/dense/kernel" input: "sub_124" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_576" op: "Assign" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m" input: "add_420" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_577" op: "Assign" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v" input: "add_421" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/attention/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/output/dense/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/attention/output/dense/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_7/attention/output/dense/bias/adam_m" input: "bert/encoder/layer_7/attention/output/dense/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/attention/output/dense/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_7/attention/output/dense/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/output/dense/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/attention/output/dense/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_7/attention/output/dense/bias/adam_v" input: "bert/encoder/layer_7/attention/output/dense/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/attention/output/dense/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_7/attention/output/dense/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_673/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_673" op: "Mul" input: "Mul_673/x" input: "bert/encoder/layer_7/attention/output/dense/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_674/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_674" op: "Mul" input: "Mul_674/x" input: "clip_by_global_norm/clip_by_global_norm/_124" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_424" op: "Add" input: "Mul_673" input: "Mul_674" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_675/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_675" op: "Mul" input: "Mul_675/x" input: "bert/encoder/layer_7/attention/output/dense/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_124" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_124" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_676/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_676" op: "Mul" input: "Mul_676/x" input: "Square_124" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_425" op: "Add" input: "Mul_675" input: "Mul_676" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_124" op: "Sqrt" input: "add_425" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_426/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_426" op: "Add" input: "Sqrt_124" input: "add_426/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_125" op: "RealDiv" input: "add_424" input: "add_426" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_677" op: "Mul" input: "add_2" input: "truediv_125" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_125" op: "Sub" input: "bert/encoder/layer_7/attention/output/dense/bias/read" input: "mul_677" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_578" op: "Assign" input: "bert/encoder/layer_7/attention/output/dense/bias" input: "sub_125" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_579" op: "Assign" input: "bert/encoder/layer_7/attention/output/dense/bias/adam_m" input: "add_424" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_580" op: "Assign" input: "bert/encoder/layer_7/attention/output/dense/bias/adam_v" input: "add_425" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_m/read" op: "Identity" input: "bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_v/read" op: "Identity" input: "bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_678/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_678" op: "Mul" input: "Mul_678/x" input: "bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_679/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_679" op: "Mul" input: "Mul_679/x" input: "clip_by_global_norm/clip_by_global_norm/_125" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_427" op: "Add" input: "Mul_678" input: "Mul_679" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_680/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_680" op: "Mul" input: "Mul_680/x" input: "bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_125" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_125" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_681/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_681" op: "Mul" input: "Mul_681/x" input: "Square_125" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_428" op: "Add" input: "Mul_680" input: "Mul_681" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_125" op: "Sqrt" input: "add_428" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_429/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_429" op: "Add" input: "Sqrt_125" input: "add_429/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_126" op: "RealDiv" input: "add_427" input: "add_429" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_682" op: "Mul" input: "add_2" input: "truediv_126" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_126" op: "Sub" input: "bert/encoder/layer_7/attention/output/LayerNorm/beta/read" input: "mul_682" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_581" op: "Assign" input: "bert/encoder/layer_7/attention/output/LayerNorm/beta" input: "sub_126" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_582" op: "Assign" input: "bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_m" input: "add_427" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_583" op: "Assign" input: "bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_v" input: "add_428" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m/read" op: "Identity" input: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v/read" op: "Identity" input: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_683/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_683" op: "Mul" input: "Mul_683/x" input: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_684/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_684" op: "Mul" input: "Mul_684/x" input: "clip_by_global_norm/clip_by_global_norm/_126" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_430" op: "Add" input: "Mul_683" input: "Mul_684" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_685/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_685" op: "Mul" input: "Mul_685/x" input: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_126" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_126" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_686/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_686" op: "Mul" input: "Mul_686/x" input: "Square_126" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_431" op: "Add" input: "Mul_685" input: "Mul_686" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_126" op: "Sqrt" input: "add_431" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_432/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_432" op: "Add" input: "Sqrt_126" input: "add_432/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_127" op: "RealDiv" input: "add_430" input: "add_432" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_687" op: "Mul" input: "add_2" input: "truediv_127" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_127" op: "Sub" input: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/read" input: "mul_687" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_584" op: "Assign" input: "bert/encoder/layer_7/attention/output/LayerNorm/gamma" input: "sub_127" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_585" op: "Assign" input: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m" input: "add_430" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_586" op: "Assign" input: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v" input: "add_431" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_688/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_688" op: "Mul" input: "Mul_688/x" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_689/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_689" op: "Mul" input: "Mul_689/x" input: "clip_by_global_norm/clip_by_global_norm/_127" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_433" op: "Add" input: "Mul_688" input: "Mul_689" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_690/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_690" op: "Mul" input: "Mul_690/x" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Square_127" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_127" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_691/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_691" op: "Mul" input: "Mul_691/x" input: "Square_127" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_434" op: "Add" input: "Mul_690" input: "Mul_691" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Sqrt_127" op: "Sqrt" input: "add_434" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_435/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_435" op: "Add" input: "Sqrt_127" input: "add_435/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "truediv_128" op: "RealDiv" input: "add_433" input: "add_435" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "mul_692/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_692" op: "Mul" input: "mul_692/x" input: "bert/encoder/layer_7/intermediate/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_436" op: "Add" input: "truediv_128" input: "mul_692" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "mul_693" op: "Mul" input: "add_2" input: "add_436" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "sub_128" op: "Sub" input: "bert/encoder/layer_7/intermediate/dense/kernel/read" input: "mul_693" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Assign_587" op: "Assign" input: "bert/encoder/layer_7/intermediate/dense/kernel" input: "sub_128" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_588" op: "Assign" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m" input: "add_433" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_589" op: "Assign" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v" input: "add_434" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_7/intermediate/dense/bias/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/intermediate/dense/bias/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_7/intermediate/dense/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/intermediate/dense/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_m" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/intermediate/dense/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_7/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_7/intermediate/dense/bias/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/intermediate/dense/bias/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_7/intermediate/dense/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/intermediate/dense/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_v" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/intermediate/dense/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_694/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_694" op: "Mul" input: "Mul_694/x" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_695/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_695" op: "Mul" input: "Mul_695/x" input: "clip_by_global_norm/clip_by_global_norm/_128" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_437" op: "Add" input: "Mul_694" input: "Mul_695" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_696/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_696" op: "Mul" input: "Mul_696/x" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Square_128" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_128" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_697/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_697" op: "Mul" input: "Mul_697/x" input: "Square_128" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_438" op: "Add" input: "Mul_696" input: "Mul_697" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Sqrt_128" op: "Sqrt" input: "add_438" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_439/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_439" op: "Add" input: "Sqrt_128" input: "add_439/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "truediv_129" op: "RealDiv" input: "add_437" input: "add_439" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "mul_698" op: "Mul" input: "add_2" input: "truediv_129" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "sub_129" op: "Sub" input: "bert/encoder/layer_7/intermediate/dense/bias/read" input: "mul_698" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Assign_590" op: "Assign" input: "bert/encoder/layer_7/intermediate/dense/bias" input: "sub_129" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_591" op: "Assign" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_m" input: "add_437" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_592" op: "Assign" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_v" input: "add_438" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_7/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_7/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_7/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_7/output/dense/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/output/dense/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_7/output/dense/kernel/adam_m" input: "bert/encoder/layer_7/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/output/dense/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_7/output/dense/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_7/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_7/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_7/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_7/output/dense/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/output/dense/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_7/output/dense/kernel/adam_v" input: "bert/encoder/layer_7/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/output/dense/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_7/output/dense/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_699/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_699" op: "Mul" input: "Mul_699/x" input: "bert/encoder/layer_7/output/dense/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_700/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_700" op: "Mul" input: "Mul_700/x" input: "clip_by_global_norm/clip_by_global_norm/_129" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_440" op: "Add" input: "Mul_699" input: "Mul_700" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_701/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_701" op: "Mul" input: "Mul_701/x" input: "bert/encoder/layer_7/output/dense/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Square_129" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_129" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_702/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_702" op: "Mul" input: "Mul_702/x" input: "Square_129" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_441" op: "Add" input: "Mul_701" input: "Mul_702" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Sqrt_129" op: "Sqrt" input: "add_441" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_442/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_442" op: "Add" input: "Sqrt_129" input: "add_442/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "truediv_130" op: "RealDiv" input: "add_440" input: "add_442" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "mul_703/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_703" op: "Mul" input: "mul_703/x" input: "bert/encoder/layer_7/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_443" op: "Add" input: "truediv_130" input: "mul_703" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "mul_704" op: "Mul" input: "add_2" input: "add_443" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "sub_130" op: "Sub" input: "bert/encoder/layer_7/output/dense/kernel/read" input: "mul_704" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Assign_593" op: "Assign" input: "bert/encoder/layer_7/output/dense/kernel" input: "sub_130" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_594" op: "Assign" input: "bert/encoder/layer_7/output/dense/kernel/adam_m" input: "add_440" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_595" op: "Assign" input: "bert/encoder/layer_7/output/dense/kernel/adam_v" input: "add_441" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/output/dense/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/output/dense/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_7/output/dense/bias/adam_m" input: "bert/encoder/layer_7/output/dense/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/output/dense/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_7/output/dense/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/output/dense/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/output/dense/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_7/output/dense/bias/adam_v" input: "bert/encoder/layer_7/output/dense/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/output/dense/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_7/output/dense/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_705/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_705" op: "Mul" input: "Mul_705/x" input: "bert/encoder/layer_7/output/dense/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_706/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_706" op: "Mul" input: "Mul_706/x" input: "clip_by_global_norm/clip_by_global_norm/_130" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_444" op: "Add" input: "Mul_705" input: "Mul_706" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_707/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_707" op: "Mul" input: "Mul_707/x" input: "bert/encoder/layer_7/output/dense/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_130" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_130" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_708/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_708" op: "Mul" input: "Mul_708/x" input: "Square_130" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_445" op: "Add" input: "Mul_707" input: "Mul_708" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_130" op: "Sqrt" input: "add_445" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_446/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_446" op: "Add" input: "Sqrt_130" input: "add_446/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_131" op: "RealDiv" input: "add_444" input: "add_446" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_709" op: "Mul" input: "add_2" input: "truediv_131" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_131" op: "Sub" input: "bert/encoder/layer_7/output/dense/bias/read" input: "mul_709" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_596" op: "Assign" input: "bert/encoder/layer_7/output/dense/bias" input: "sub_131" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_597" op: "Assign" input: "bert/encoder/layer_7/output/dense/bias/adam_m" input: "add_444" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_598" op: "Assign" input: "bert/encoder/layer_7/output/dense/bias/adam_v" input: "add_445" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/output/LayerNorm/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/output/LayerNorm/beta/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/output/LayerNorm/beta/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_7/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_7/output/LayerNorm/beta/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/output/LayerNorm/beta/adam_m/read" op: "Identity" input: "bert/encoder/layer_7/output/LayerNorm/beta/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/output/LayerNorm/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/output/LayerNorm/beta/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/output/LayerNorm/beta/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_7/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_7/output/LayerNorm/beta/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/output/LayerNorm/beta/adam_v/read" op: "Identity" input: "bert/encoder/layer_7/output/LayerNorm/beta/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_710/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_710" op: "Mul" input: "Mul_710/x" input: "bert/encoder/layer_7/output/LayerNorm/beta/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_711/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_711" op: "Mul" input: "Mul_711/x" input: "clip_by_global_norm/clip_by_global_norm/_131" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_447" op: "Add" input: "Mul_710" input: "Mul_711" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_712/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_712" op: "Mul" input: "Mul_712/x" input: "bert/encoder/layer_7/output/LayerNorm/beta/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_131" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_131" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_713/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_713" op: "Mul" input: "Mul_713/x" input: "Square_131" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_448" op: "Add" input: "Mul_712" input: "Mul_713" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_131" op: "Sqrt" input: "add_448" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_449/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_449" op: "Add" input: "Sqrt_131" input: "add_449/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_132" op: "RealDiv" input: "add_447" input: "add_449" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_714" op: "Mul" input: "add_2" input: "truediv_132" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_132" op: "Sub" input: "bert/encoder/layer_7/output/LayerNorm/beta/read" input: "mul_714" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_599" op: "Assign" input: "bert/encoder/layer_7/output/LayerNorm/beta" input: "sub_132" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_600" op: "Assign" input: "bert/encoder/layer_7/output/LayerNorm/beta/adam_m" input: "add_447" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_601" op: "Assign" input: "bert/encoder/layer_7/output/LayerNorm/beta/adam_v" input: "add_448" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/output/LayerNorm/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/output/LayerNorm/gamma/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/output/LayerNorm/gamma/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_7/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_7/output/LayerNorm/gamma/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/output/LayerNorm/gamma/adam_m/read" op: "Identity" input: "bert/encoder/layer_7/output/LayerNorm/gamma/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/output/LayerNorm/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/output/LayerNorm/gamma/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_7/output/LayerNorm/gamma/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_7/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_7/output/LayerNorm/gamma/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_7/output/LayerNorm/gamma/adam_v/read" op: "Identity" input: "bert/encoder/layer_7/output/LayerNorm/gamma/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_715/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_715" op: "Mul" input: "Mul_715/x" input: "bert/encoder/layer_7/output/LayerNorm/gamma/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_716/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_716" op: "Mul" input: "Mul_716/x" input: "clip_by_global_norm/clip_by_global_norm/_132" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_450" op: "Add" input: "Mul_715" input: "Mul_716" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_717/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_717" op: "Mul" input: "Mul_717/x" input: "bert/encoder/layer_7/output/LayerNorm/gamma/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_132" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_132" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_718/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_718" op: "Mul" input: "Mul_718/x" input: "Square_132" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_451" op: "Add" input: "Mul_717" input: "Mul_718" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_132" op: "Sqrt" input: "add_451" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_452/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_452" op: "Add" input: "Sqrt_132" input: "add_452/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_133" op: "RealDiv" input: "add_450" input: "add_452" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_719" op: "Mul" input: "add_2" input: "truediv_133" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_133" op: "Sub" input: "bert/encoder/layer_7/output/LayerNorm/gamma/read" input: "mul_719" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_602" op: "Assign" input: "bert/encoder/layer_7/output/LayerNorm/gamma" input: "sub_133" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_603" op: "Assign" input: "bert/encoder/layer_7/output/LayerNorm/gamma/adam_m" input: "add_450" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_604" op: "Assign" input: "bert/encoder/layer_7/output/LayerNorm/gamma/adam_v" input: "add_451" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_m" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_v" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_720/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_720" op: "Mul" input: "Mul_720/x" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_721/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_721" op: "Mul" input: "Mul_721/x" input: "clip_by_global_norm/clip_by_global_norm/_133" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_453" op: "Add" input: "Mul_720" input: "Mul_721" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_722/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_722" op: "Mul" input: "Mul_722/x" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_133" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_133" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_723/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_723" op: "Mul" input: "Mul_723/x" input: "Square_133" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_454" op: "Add" input: "Mul_722" input: "Mul_723" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_133" op: "Sqrt" input: "add_454" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_455/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_455" op: "Add" input: "Sqrt_133" input: "add_455/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_134" op: "RealDiv" input: "add_453" input: "add_455" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_724/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_724" op: "Mul" input: "mul_724/x" input: "bert/encoder/layer_8/attention/self/query/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_456" op: "Add" input: "truediv_134" input: "mul_724" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_725" op: "Mul" input: "add_2" input: "add_456" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_134" op: "Sub" input: "bert/encoder/layer_8/attention/self/query/kernel/read" input: "mul_725" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_605" op: "Assign" input: "bert/encoder/layer_8/attention/self/query/kernel" input: "sub_134" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_606" op: "Assign" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_m" input: "add_453" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_607" op: "Assign" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_v" input: "add_454" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/attention/self/query/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/self/query/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/attention/self/query/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_8/attention/self/query/bias/adam_m" input: "bert/encoder/layer_8/attention/self/query/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/attention/self/query/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_8/attention/self/query/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/self/query/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/self/query/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/attention/self/query/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_8/attention/self/query/bias/adam_v" input: "bert/encoder/layer_8/attention/self/query/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/attention/self/query/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_8/attention/self/query/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_726/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_726" op: "Mul" input: "Mul_726/x" input: "bert/encoder/layer_8/attention/self/query/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_727/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_727" op: "Mul" input: "Mul_727/x" input: "clip_by_global_norm/clip_by_global_norm/_134" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_457" op: "Add" input: "Mul_726" input: "Mul_727" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_728/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_728" op: "Mul" input: "Mul_728/x" input: "bert/encoder/layer_8/attention/self/query/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_134" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_134" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_729/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_729" op: "Mul" input: "Mul_729/x" input: "Square_134" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_458" op: "Add" input: "Mul_728" input: "Mul_729" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_134" op: "Sqrt" input: "add_458" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_459/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_459" op: "Add" input: "Sqrt_134" input: "add_459/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_135" op: "RealDiv" input: "add_457" input: "add_459" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_730" op: "Mul" input: "add_2" input: "truediv_135" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_135" op: "Sub" input: "bert/encoder/layer_8/attention/self/query/bias/read" input: "mul_730" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_608" op: "Assign" input: "bert/encoder/layer_8/attention/self/query/bias" input: "sub_135" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_609" op: "Assign" input: "bert/encoder/layer_8/attention/self/query/bias/adam_m" input: "add_457" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_610" op: "Assign" input: "bert/encoder/layer_8/attention/self/query/bias/adam_v" input: "add_458" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_m" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_v" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_731/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_731" op: "Mul" input: "Mul_731/x" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_732/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_732" op: "Mul" input: "Mul_732/x" input: "clip_by_global_norm/clip_by_global_norm/_135" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_460" op: "Add" input: "Mul_731" input: "Mul_732" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_733/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_733" op: "Mul" input: "Mul_733/x" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_135" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_135" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_734/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_734" op: "Mul" input: "Mul_734/x" input: "Square_135" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_461" op: "Add" input: "Mul_733" input: "Mul_734" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_135" op: "Sqrt" input: "add_461" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_462/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_462" op: "Add" input: "Sqrt_135" input: "add_462/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_136" op: "RealDiv" input: "add_460" input: "add_462" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_735/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_735" op: "Mul" input: "mul_735/x" input: "bert/encoder/layer_8/attention/self/key/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_463" op: "Add" input: "truediv_136" input: "mul_735" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_736" op: "Mul" input: "add_2" input: "add_463" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_136" op: "Sub" input: "bert/encoder/layer_8/attention/self/key/kernel/read" input: "mul_736" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_611" op: "Assign" input: "bert/encoder/layer_8/attention/self/key/kernel" input: "sub_136" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_612" op: "Assign" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_m" input: "add_460" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_613" op: "Assign" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_v" input: "add_461" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/attention/self/key/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/self/key/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/attention/self/key/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_8/attention/self/key/bias/adam_m" input: "bert/encoder/layer_8/attention/self/key/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/attention/self/key/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_8/attention/self/key/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/self/key/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/self/key/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/attention/self/key/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_8/attention/self/key/bias/adam_v" input: "bert/encoder/layer_8/attention/self/key/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/attention/self/key/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_8/attention/self/key/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_737/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_737" op: "Mul" input: "Mul_737/x" input: "bert/encoder/layer_8/attention/self/key/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_738/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_738" op: "Mul" input: "Mul_738/x" input: "clip_by_global_norm/clip_by_global_norm/_136" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_464" op: "Add" input: "Mul_737" input: "Mul_738" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_739/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_739" op: "Mul" input: "Mul_739/x" input: "bert/encoder/layer_8/attention/self/key/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_136" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_136" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_740/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_740" op: "Mul" input: "Mul_740/x" input: "Square_136" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_465" op: "Add" input: "Mul_739" input: "Mul_740" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_136" op: "Sqrt" input: "add_465" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_466/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_466" op: "Add" input: "Sqrt_136" input: "add_466/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_137" op: "RealDiv" input: "add_464" input: "add_466" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_741" op: "Mul" input: "add_2" input: "truediv_137" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_137" op: "Sub" input: "bert/encoder/layer_8/attention/self/key/bias/read" input: "mul_741" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_614" op: "Assign" input: "bert/encoder/layer_8/attention/self/key/bias" input: "sub_137" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_615" op: "Assign" input: "bert/encoder/layer_8/attention/self/key/bias/adam_m" input: "add_464" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_616" op: "Assign" input: "bert/encoder/layer_8/attention/self/key/bias/adam_v" input: "add_465" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_m" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_v" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_742/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_742" op: "Mul" input: "Mul_742/x" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_743/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_743" op: "Mul" input: "Mul_743/x" input: "clip_by_global_norm/clip_by_global_norm/_137" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_467" op: "Add" input: "Mul_742" input: "Mul_743" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_744/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_744" op: "Mul" input: "Mul_744/x" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_137" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_137" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_745/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_745" op: "Mul" input: "Mul_745/x" input: "Square_137" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_468" op: "Add" input: "Mul_744" input: "Mul_745" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_137" op: "Sqrt" input: "add_468" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_469/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_469" op: "Add" input: "Sqrt_137" input: "add_469/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_138" op: "RealDiv" input: "add_467" input: "add_469" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_746/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_746" op: "Mul" input: "mul_746/x" input: "bert/encoder/layer_8/attention/self/value/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_470" op: "Add" input: "truediv_138" input: "mul_746" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_747" op: "Mul" input: "add_2" input: "add_470" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_138" op: "Sub" input: "bert/encoder/layer_8/attention/self/value/kernel/read" input: "mul_747" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_617" op: "Assign" input: "bert/encoder/layer_8/attention/self/value/kernel" input: "sub_138" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_618" op: "Assign" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_m" input: "add_467" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_619" op: "Assign" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_v" input: "add_468" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/attention/self/value/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/self/value/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/attention/self/value/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_8/attention/self/value/bias/adam_m" input: "bert/encoder/layer_8/attention/self/value/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/attention/self/value/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_8/attention/self/value/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/self/value/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/self/value/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/attention/self/value/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_8/attention/self/value/bias/adam_v" input: "bert/encoder/layer_8/attention/self/value/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/attention/self/value/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_8/attention/self/value/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_748/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_748" op: "Mul" input: "Mul_748/x" input: "bert/encoder/layer_8/attention/self/value/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_749/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_749" op: "Mul" input: "Mul_749/x" input: "clip_by_global_norm/clip_by_global_norm/_138" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_471" op: "Add" input: "Mul_748" input: "Mul_749" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_750/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_750" op: "Mul" input: "Mul_750/x" input: "bert/encoder/layer_8/attention/self/value/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_138" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_138" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_751/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_751" op: "Mul" input: "Mul_751/x" input: "Square_138" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_472" op: "Add" input: "Mul_750" input: "Mul_751" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_138" op: "Sqrt" input: "add_472" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_473/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_473" op: "Add" input: "Sqrt_138" input: "add_473/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_139" op: "RealDiv" input: "add_471" input: "add_473" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_752" op: "Mul" input: "add_2" input: "truediv_139" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_139" op: "Sub" input: "bert/encoder/layer_8/attention/self/value/bias/read" input: "mul_752" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_620" op: "Assign" input: "bert/encoder/layer_8/attention/self/value/bias" input: "sub_139" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_621" op: "Assign" input: "bert/encoder/layer_8/attention/self/value/bias/adam_m" input: "add_471" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_622" op: "Assign" input: "bert/encoder/layer_8/attention/self/value/bias/adam_v" input: "add_472" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_753/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_753" op: "Mul" input: "Mul_753/x" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_754/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_754" op: "Mul" input: "Mul_754/x" input: "clip_by_global_norm/clip_by_global_norm/_139" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_474" op: "Add" input: "Mul_753" input: "Mul_754" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_755/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_755" op: "Mul" input: "Mul_755/x" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_139" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_139" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_756/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_756" op: "Mul" input: "Mul_756/x" input: "Square_139" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_475" op: "Add" input: "Mul_755" input: "Mul_756" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_139" op: "Sqrt" input: "add_475" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_476/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_476" op: "Add" input: "Sqrt_139" input: "add_476/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_140" op: "RealDiv" input: "add_474" input: "add_476" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_757/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_757" op: "Mul" input: "mul_757/x" input: "bert/encoder/layer_8/attention/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_477" op: "Add" input: "truediv_140" input: "mul_757" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_758" op: "Mul" input: "add_2" input: "add_477" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_140" op: "Sub" input: "bert/encoder/layer_8/attention/output/dense/kernel/read" input: "mul_758" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_623" op: "Assign" input: "bert/encoder/layer_8/attention/output/dense/kernel" input: "sub_140" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_624" op: "Assign" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m" input: "add_474" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_625" op: "Assign" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v" input: "add_475" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/attention/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/output/dense/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/attention/output/dense/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_8/attention/output/dense/bias/adam_m" input: "bert/encoder/layer_8/attention/output/dense/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/attention/output/dense/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_8/attention/output/dense/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/output/dense/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/attention/output/dense/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_8/attention/output/dense/bias/adam_v" input: "bert/encoder/layer_8/attention/output/dense/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/attention/output/dense/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_8/attention/output/dense/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_759/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_759" op: "Mul" input: "Mul_759/x" input: "bert/encoder/layer_8/attention/output/dense/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_760/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_760" op: "Mul" input: "Mul_760/x" input: "clip_by_global_norm/clip_by_global_norm/_140" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_478" op: "Add" input: "Mul_759" input: "Mul_760" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_761/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_761" op: "Mul" input: "Mul_761/x" input: "bert/encoder/layer_8/attention/output/dense/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_140" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_140" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_762/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_762" op: "Mul" input: "Mul_762/x" input: "Square_140" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_479" op: "Add" input: "Mul_761" input: "Mul_762" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_140" op: "Sqrt" input: "add_479" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_480/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_480" op: "Add" input: "Sqrt_140" input: "add_480/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_141" op: "RealDiv" input: "add_478" input: "add_480" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_763" op: "Mul" input: "add_2" input: "truediv_141" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_141" op: "Sub" input: "bert/encoder/layer_8/attention/output/dense/bias/read" input: "mul_763" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_626" op: "Assign" input: "bert/encoder/layer_8/attention/output/dense/bias" input: "sub_141" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_627" op: "Assign" input: "bert/encoder/layer_8/attention/output/dense/bias/adam_m" input: "add_478" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_628" op: "Assign" input: "bert/encoder/layer_8/attention/output/dense/bias/adam_v" input: "add_479" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_m/read" op: "Identity" input: "bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_v/read" op: "Identity" input: "bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_764/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_764" op: "Mul" input: "Mul_764/x" input: "bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_765/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_765" op: "Mul" input: "Mul_765/x" input: "clip_by_global_norm/clip_by_global_norm/_141" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_481" op: "Add" input: "Mul_764" input: "Mul_765" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_766/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_766" op: "Mul" input: "Mul_766/x" input: "bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_141" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_141" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_767/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_767" op: "Mul" input: "Mul_767/x" input: "Square_141" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_482" op: "Add" input: "Mul_766" input: "Mul_767" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_141" op: "Sqrt" input: "add_482" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_483/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_483" op: "Add" input: "Sqrt_141" input: "add_483/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_142" op: "RealDiv" input: "add_481" input: "add_483" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_768" op: "Mul" input: "add_2" input: "truediv_142" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_142" op: "Sub" input: "bert/encoder/layer_8/attention/output/LayerNorm/beta/read" input: "mul_768" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_629" op: "Assign" input: "bert/encoder/layer_8/attention/output/LayerNorm/beta" input: "sub_142" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_630" op: "Assign" input: "bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_m" input: "add_481" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_631" op: "Assign" input: "bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_v" input: "add_482" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m/read" op: "Identity" input: "bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v/read" op: "Identity" input: "bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_769/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_769" op: "Mul" input: "Mul_769/x" input: "bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_770/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_770" op: "Mul" input: "Mul_770/x" input: "clip_by_global_norm/clip_by_global_norm/_142" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_484" op: "Add" input: "Mul_769" input: "Mul_770" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_771/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_771" op: "Mul" input: "Mul_771/x" input: "bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_142" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_142" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_772/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_772" op: "Mul" input: "Mul_772/x" input: "Square_142" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_485" op: "Add" input: "Mul_771" input: "Mul_772" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_142" op: "Sqrt" input: "add_485" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_486/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_486" op: "Add" input: "Sqrt_142" input: "add_486/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_143" op: "RealDiv" input: "add_484" input: "add_486" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_773" op: "Mul" input: "add_2" input: "truediv_143" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_143" op: "Sub" input: "bert/encoder/layer_8/attention/output/LayerNorm/gamma/read" input: "mul_773" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_632" op: "Assign" input: "bert/encoder/layer_8/attention/output/LayerNorm/gamma" input: "sub_143" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_633" op: "Assign" input: "bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m" input: "add_484" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_634" op: "Assign" input: "bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v" input: "add_485" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_774/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_774" op: "Mul" input: "Mul_774/x" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_775/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_775" op: "Mul" input: "Mul_775/x" input: "clip_by_global_norm/clip_by_global_norm/_143" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_487" op: "Add" input: "Mul_774" input: "Mul_775" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_776/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_776" op: "Mul" input: "Mul_776/x" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Square_143" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_143" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_777/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_777" op: "Mul" input: "Mul_777/x" input: "Square_143" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_488" op: "Add" input: "Mul_776" input: "Mul_777" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Sqrt_143" op: "Sqrt" input: "add_488" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_489/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_489" op: "Add" input: "Sqrt_143" input: "add_489/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "truediv_144" op: "RealDiv" input: "add_487" input: "add_489" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "mul_778/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_778" op: "Mul" input: "mul_778/x" input: "bert/encoder/layer_8/intermediate/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_490" op: "Add" input: "truediv_144" input: "mul_778" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "mul_779" op: "Mul" input: "add_2" input: "add_490" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "sub_144" op: "Sub" input: "bert/encoder/layer_8/intermediate/dense/kernel/read" input: "mul_779" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Assign_635" op: "Assign" input: "bert/encoder/layer_8/intermediate/dense/kernel" input: "sub_144" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_636" op: "Assign" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m" input: "add_487" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_637" op: "Assign" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v" input: "add_488" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_8/intermediate/dense/bias/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/intermediate/dense/bias/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_8/intermediate/dense/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/intermediate/dense/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_m" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/intermediate/dense/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_8/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_8/intermediate/dense/bias/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/intermediate/dense/bias/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_8/intermediate/dense/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/intermediate/dense/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_v" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/intermediate/dense/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_780/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_780" op: "Mul" input: "Mul_780/x" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_781/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_781" op: "Mul" input: "Mul_781/x" input: "clip_by_global_norm/clip_by_global_norm/_144" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_491" op: "Add" input: "Mul_780" input: "Mul_781" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_782/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_782" op: "Mul" input: "Mul_782/x" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Square_144" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_144" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_783/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_783" op: "Mul" input: "Mul_783/x" input: "Square_144" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_492" op: "Add" input: "Mul_782" input: "Mul_783" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Sqrt_144" op: "Sqrt" input: "add_492" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_493/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_493" op: "Add" input: "Sqrt_144" input: "add_493/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "truediv_145" op: "RealDiv" input: "add_491" input: "add_493" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "mul_784" op: "Mul" input: "add_2" input: "truediv_145" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "sub_145" op: "Sub" input: "bert/encoder/layer_8/intermediate/dense/bias/read" input: "mul_784" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Assign_638" op: "Assign" input: "bert/encoder/layer_8/intermediate/dense/bias" input: "sub_145" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_639" op: "Assign" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_m" input: "add_491" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_640" op: "Assign" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_v" input: "add_492" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_8/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_8/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_8/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_8/output/dense/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/output/dense/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_8/output/dense/kernel/adam_m" input: "bert/encoder/layer_8/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/output/dense/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_8/output/dense/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_8/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_8/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_8/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_8/output/dense/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/output/dense/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_8/output/dense/kernel/adam_v" input: "bert/encoder/layer_8/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/output/dense/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_8/output/dense/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_785/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_785" op: "Mul" input: "Mul_785/x" input: "bert/encoder/layer_8/output/dense/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_786/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_786" op: "Mul" input: "Mul_786/x" input: "clip_by_global_norm/clip_by_global_norm/_145" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_494" op: "Add" input: "Mul_785" input: "Mul_786" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_787/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_787" op: "Mul" input: "Mul_787/x" input: "bert/encoder/layer_8/output/dense/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Square_145" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_145" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_788/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_788" op: "Mul" input: "Mul_788/x" input: "Square_145" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_495" op: "Add" input: "Mul_787" input: "Mul_788" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Sqrt_145" op: "Sqrt" input: "add_495" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_496/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_496" op: "Add" input: "Sqrt_145" input: "add_496/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "truediv_146" op: "RealDiv" input: "add_494" input: "add_496" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "mul_789/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_789" op: "Mul" input: "mul_789/x" input: "bert/encoder/layer_8/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_497" op: "Add" input: "truediv_146" input: "mul_789" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "mul_790" op: "Mul" input: "add_2" input: "add_497" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "sub_146" op: "Sub" input: "bert/encoder/layer_8/output/dense/kernel/read" input: "mul_790" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Assign_641" op: "Assign" input: "bert/encoder/layer_8/output/dense/kernel" input: "sub_146" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_642" op: "Assign" input: "bert/encoder/layer_8/output/dense/kernel/adam_m" input: "add_494" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_643" op: "Assign" input: "bert/encoder/layer_8/output/dense/kernel/adam_v" input: "add_495" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/output/dense/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/output/dense/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_8/output/dense/bias/adam_m" input: "bert/encoder/layer_8/output/dense/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/output/dense/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_8/output/dense/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/output/dense/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/output/dense/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_8/output/dense/bias/adam_v" input: "bert/encoder/layer_8/output/dense/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/output/dense/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_8/output/dense/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_791/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_791" op: "Mul" input: "Mul_791/x" input: "bert/encoder/layer_8/output/dense/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_792/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_792" op: "Mul" input: "Mul_792/x" input: "clip_by_global_norm/clip_by_global_norm/_146" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_498" op: "Add" input: "Mul_791" input: "Mul_792" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_793/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_793" op: "Mul" input: "Mul_793/x" input: "bert/encoder/layer_8/output/dense/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_146" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_146" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_794/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_794" op: "Mul" input: "Mul_794/x" input: "Square_146" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_499" op: "Add" input: "Mul_793" input: "Mul_794" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_146" op: "Sqrt" input: "add_499" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_500/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_500" op: "Add" input: "Sqrt_146" input: "add_500/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_147" op: "RealDiv" input: "add_498" input: "add_500" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_795" op: "Mul" input: "add_2" input: "truediv_147" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_147" op: "Sub" input: "bert/encoder/layer_8/output/dense/bias/read" input: "mul_795" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_644" op: "Assign" input: "bert/encoder/layer_8/output/dense/bias" input: "sub_147" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_645" op: "Assign" input: "bert/encoder/layer_8/output/dense/bias/adam_m" input: "add_498" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_646" op: "Assign" input: "bert/encoder/layer_8/output/dense/bias/adam_v" input: "add_499" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/output/LayerNorm/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/output/LayerNorm/beta/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/output/LayerNorm/beta/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_8/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_8/output/LayerNorm/beta/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/output/LayerNorm/beta/adam_m/read" op: "Identity" input: "bert/encoder/layer_8/output/LayerNorm/beta/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/output/LayerNorm/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/output/LayerNorm/beta/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/output/LayerNorm/beta/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_8/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_8/output/LayerNorm/beta/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/output/LayerNorm/beta/adam_v/read" op: "Identity" input: "bert/encoder/layer_8/output/LayerNorm/beta/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_796/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_796" op: "Mul" input: "Mul_796/x" input: "bert/encoder/layer_8/output/LayerNorm/beta/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_797/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_797" op: "Mul" input: "Mul_797/x" input: "clip_by_global_norm/clip_by_global_norm/_147" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_501" op: "Add" input: "Mul_796" input: "Mul_797" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_798/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_798" op: "Mul" input: "Mul_798/x" input: "bert/encoder/layer_8/output/LayerNorm/beta/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_147" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_147" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_799/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_799" op: "Mul" input: "Mul_799/x" input: "Square_147" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_502" op: "Add" input: "Mul_798" input: "Mul_799" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_147" op: "Sqrt" input: "add_502" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_503/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_503" op: "Add" input: "Sqrt_147" input: "add_503/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_148" op: "RealDiv" input: "add_501" input: "add_503" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_800" op: "Mul" input: "add_2" input: "truediv_148" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_148" op: "Sub" input: "bert/encoder/layer_8/output/LayerNorm/beta/read" input: "mul_800" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_647" op: "Assign" input: "bert/encoder/layer_8/output/LayerNorm/beta" input: "sub_148" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_648" op: "Assign" input: "bert/encoder/layer_8/output/LayerNorm/beta/adam_m" input: "add_501" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_649" op: "Assign" input: "bert/encoder/layer_8/output/LayerNorm/beta/adam_v" input: "add_502" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/output/LayerNorm/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/output/LayerNorm/gamma/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/output/LayerNorm/gamma/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_8/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_8/output/LayerNorm/gamma/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/output/LayerNorm/gamma/adam_m/read" op: "Identity" input: "bert/encoder/layer_8/output/LayerNorm/gamma/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/output/LayerNorm/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/output/LayerNorm/gamma/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_8/output/LayerNorm/gamma/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_8/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_8/output/LayerNorm/gamma/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_8/output/LayerNorm/gamma/adam_v/read" op: "Identity" input: "bert/encoder/layer_8/output/LayerNorm/gamma/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_801/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_801" op: "Mul" input: "Mul_801/x" input: "bert/encoder/layer_8/output/LayerNorm/gamma/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_802/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_802" op: "Mul" input: "Mul_802/x" input: "clip_by_global_norm/clip_by_global_norm/_148" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_504" op: "Add" input: "Mul_801" input: "Mul_802" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_803/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_803" op: "Mul" input: "Mul_803/x" input: "bert/encoder/layer_8/output/LayerNorm/gamma/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_148" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_148" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_804/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_804" op: "Mul" input: "Mul_804/x" input: "Square_148" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_505" op: "Add" input: "Mul_803" input: "Mul_804" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_148" op: "Sqrt" input: "add_505" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_506/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_506" op: "Add" input: "Sqrt_148" input: "add_506/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_149" op: "RealDiv" input: "add_504" input: "add_506" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_805" op: "Mul" input: "add_2" input: "truediv_149" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_149" op: "Sub" input: "bert/encoder/layer_8/output/LayerNorm/gamma/read" input: "mul_805" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_650" op: "Assign" input: "bert/encoder/layer_8/output/LayerNorm/gamma" input: "sub_149" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_651" op: "Assign" input: "bert/encoder/layer_8/output/LayerNorm/gamma/adam_m" input: "add_504" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_652" op: "Assign" input: "bert/encoder/layer_8/output/LayerNorm/gamma/adam_v" input: "add_505" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_m" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_v" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_806/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_806" op: "Mul" input: "Mul_806/x" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_807/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_807" op: "Mul" input: "Mul_807/x" input: "clip_by_global_norm/clip_by_global_norm/_149" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_507" op: "Add" input: "Mul_806" input: "Mul_807" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_808/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_808" op: "Mul" input: "Mul_808/x" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_149" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_149" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_809/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_809" op: "Mul" input: "Mul_809/x" input: "Square_149" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_508" op: "Add" input: "Mul_808" input: "Mul_809" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_149" op: "Sqrt" input: "add_508" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_509/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_509" op: "Add" input: "Sqrt_149" input: "add_509/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_150" op: "RealDiv" input: "add_507" input: "add_509" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_810/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_810" op: "Mul" input: "mul_810/x" input: "bert/encoder/layer_9/attention/self/query/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_510" op: "Add" input: "truediv_150" input: "mul_810" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_811" op: "Mul" input: "add_2" input: "add_510" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_150" op: "Sub" input: "bert/encoder/layer_9/attention/self/query/kernel/read" input: "mul_811" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_653" op: "Assign" input: "bert/encoder/layer_9/attention/self/query/kernel" input: "sub_150" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_654" op: "Assign" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_m" input: "add_507" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_655" op: "Assign" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_v" input: "add_508" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/attention/self/query/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/self/query/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/attention/self/query/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_9/attention/self/query/bias/adam_m" input: "bert/encoder/layer_9/attention/self/query/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/attention/self/query/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_9/attention/self/query/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/self/query/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/self/query/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/attention/self/query/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_9/attention/self/query/bias/adam_v" input: "bert/encoder/layer_9/attention/self/query/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/attention/self/query/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_9/attention/self/query/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_812/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_812" op: "Mul" input: "Mul_812/x" input: "bert/encoder/layer_9/attention/self/query/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_813/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_813" op: "Mul" input: "Mul_813/x" input: "clip_by_global_norm/clip_by_global_norm/_150" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_511" op: "Add" input: "Mul_812" input: "Mul_813" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_814/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_814" op: "Mul" input: "Mul_814/x" input: "bert/encoder/layer_9/attention/self/query/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_150" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_150" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_815/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_815" op: "Mul" input: "Mul_815/x" input: "Square_150" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_512" op: "Add" input: "Mul_814" input: "Mul_815" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_150" op: "Sqrt" input: "add_512" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_513/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_513" op: "Add" input: "Sqrt_150" input: "add_513/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_151" op: "RealDiv" input: "add_511" input: "add_513" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_816" op: "Mul" input: "add_2" input: "truediv_151" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_151" op: "Sub" input: "bert/encoder/layer_9/attention/self/query/bias/read" input: "mul_816" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_656" op: "Assign" input: "bert/encoder/layer_9/attention/self/query/bias" input: "sub_151" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_657" op: "Assign" input: "bert/encoder/layer_9/attention/self/query/bias/adam_m" input: "add_511" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_658" op: "Assign" input: "bert/encoder/layer_9/attention/self/query/bias/adam_v" input: "add_512" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_m" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_v" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_817/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_817" op: "Mul" input: "Mul_817/x" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_818/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_818" op: "Mul" input: "Mul_818/x" input: "clip_by_global_norm/clip_by_global_norm/_151" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_514" op: "Add" input: "Mul_817" input: "Mul_818" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_819/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_819" op: "Mul" input: "Mul_819/x" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_151" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_151" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_820/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_820" op: "Mul" input: "Mul_820/x" input: "Square_151" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_515" op: "Add" input: "Mul_819" input: "Mul_820" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_151" op: "Sqrt" input: "add_515" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_516/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_516" op: "Add" input: "Sqrt_151" input: "add_516/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_152" op: "RealDiv" input: "add_514" input: "add_516" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_821/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_821" op: "Mul" input: "mul_821/x" input: "bert/encoder/layer_9/attention/self/key/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_517" op: "Add" input: "truediv_152" input: "mul_821" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_822" op: "Mul" input: "add_2" input: "add_517" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_152" op: "Sub" input: "bert/encoder/layer_9/attention/self/key/kernel/read" input: "mul_822" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_659" op: "Assign" input: "bert/encoder/layer_9/attention/self/key/kernel" input: "sub_152" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_660" op: "Assign" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_m" input: "add_514" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_661" op: "Assign" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_v" input: "add_515" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/attention/self/key/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/self/key/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/attention/self/key/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_9/attention/self/key/bias/adam_m" input: "bert/encoder/layer_9/attention/self/key/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/attention/self/key/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_9/attention/self/key/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/self/key/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/self/key/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/attention/self/key/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_9/attention/self/key/bias/adam_v" input: "bert/encoder/layer_9/attention/self/key/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/attention/self/key/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_9/attention/self/key/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_823/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_823" op: "Mul" input: "Mul_823/x" input: "bert/encoder/layer_9/attention/self/key/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_824/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_824" op: "Mul" input: "Mul_824/x" input: "clip_by_global_norm/clip_by_global_norm/_152" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_518" op: "Add" input: "Mul_823" input: "Mul_824" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_825/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_825" op: "Mul" input: "Mul_825/x" input: "bert/encoder/layer_9/attention/self/key/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_152" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_152" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_826/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_826" op: "Mul" input: "Mul_826/x" input: "Square_152" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_519" op: "Add" input: "Mul_825" input: "Mul_826" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_152" op: "Sqrt" input: "add_519" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_520/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_520" op: "Add" input: "Sqrt_152" input: "add_520/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_153" op: "RealDiv" input: "add_518" input: "add_520" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_827" op: "Mul" input: "add_2" input: "truediv_153" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_153" op: "Sub" input: "bert/encoder/layer_9/attention/self/key/bias/read" input: "mul_827" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_662" op: "Assign" input: "bert/encoder/layer_9/attention/self/key/bias" input: "sub_153" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_663" op: "Assign" input: "bert/encoder/layer_9/attention/self/key/bias/adam_m" input: "add_518" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_664" op: "Assign" input: "bert/encoder/layer_9/attention/self/key/bias/adam_v" input: "add_519" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_m" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_v" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_828/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_828" op: "Mul" input: "Mul_828/x" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_829/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_829" op: "Mul" input: "Mul_829/x" input: "clip_by_global_norm/clip_by_global_norm/_153" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_521" op: "Add" input: "Mul_828" input: "Mul_829" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_830/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_830" op: "Mul" input: "Mul_830/x" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_153" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_153" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_831/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_831" op: "Mul" input: "Mul_831/x" input: "Square_153" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_522" op: "Add" input: "Mul_830" input: "Mul_831" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_153" op: "Sqrt" input: "add_522" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_523/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_523" op: "Add" input: "Sqrt_153" input: "add_523/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_154" op: "RealDiv" input: "add_521" input: "add_523" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_832/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_832" op: "Mul" input: "mul_832/x" input: "bert/encoder/layer_9/attention/self/value/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_524" op: "Add" input: "truediv_154" input: "mul_832" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_833" op: "Mul" input: "add_2" input: "add_524" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_154" op: "Sub" input: "bert/encoder/layer_9/attention/self/value/kernel/read" input: "mul_833" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_665" op: "Assign" input: "bert/encoder/layer_9/attention/self/value/kernel" input: "sub_154" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_666" op: "Assign" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_m" input: "add_521" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_667" op: "Assign" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_v" input: "add_522" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/attention/self/value/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/self/value/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/attention/self/value/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_9/attention/self/value/bias/adam_m" input: "bert/encoder/layer_9/attention/self/value/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/attention/self/value/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_9/attention/self/value/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/self/value/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/self/value/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/attention/self/value/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_9/attention/self/value/bias/adam_v" input: "bert/encoder/layer_9/attention/self/value/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/attention/self/value/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_9/attention/self/value/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_834/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_834" op: "Mul" input: "Mul_834/x" input: "bert/encoder/layer_9/attention/self/value/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_835/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_835" op: "Mul" input: "Mul_835/x" input: "clip_by_global_norm/clip_by_global_norm/_154" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_525" op: "Add" input: "Mul_834" input: "Mul_835" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_836/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_836" op: "Mul" input: "Mul_836/x" input: "bert/encoder/layer_9/attention/self/value/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_154" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_154" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_837/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_837" op: "Mul" input: "Mul_837/x" input: "Square_154" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_526" op: "Add" input: "Mul_836" input: "Mul_837" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_154" op: "Sqrt" input: "add_526" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_527/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_527" op: "Add" input: "Sqrt_154" input: "add_527/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_155" op: "RealDiv" input: "add_525" input: "add_527" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_838" op: "Mul" input: "add_2" input: "truediv_155" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_155" op: "Sub" input: "bert/encoder/layer_9/attention/self/value/bias/read" input: "mul_838" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_668" op: "Assign" input: "bert/encoder/layer_9/attention/self/value/bias" input: "sub_155" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_669" op: "Assign" input: "bert/encoder/layer_9/attention/self/value/bias/adam_m" input: "add_525" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_670" op: "Assign" input: "bert/encoder/layer_9/attention/self/value/bias/adam_v" input: "add_526" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_839/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_839" op: "Mul" input: "Mul_839/x" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_840/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_840" op: "Mul" input: "Mul_840/x" input: "clip_by_global_norm/clip_by_global_norm/_155" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_528" op: "Add" input: "Mul_839" input: "Mul_840" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_841/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_841" op: "Mul" input: "Mul_841/x" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_155" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_155" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_842/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_842" op: "Mul" input: "Mul_842/x" input: "Square_155" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_529" op: "Add" input: "Mul_841" input: "Mul_842" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_155" op: "Sqrt" input: "add_529" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_530/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_530" op: "Add" input: "Sqrt_155" input: "add_530/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_156" op: "RealDiv" input: "add_528" input: "add_530" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_843/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_843" op: "Mul" input: "mul_843/x" input: "bert/encoder/layer_9/attention/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_531" op: "Add" input: "truediv_156" input: "mul_843" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_844" op: "Mul" input: "add_2" input: "add_531" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_156" op: "Sub" input: "bert/encoder/layer_9/attention/output/dense/kernel/read" input: "mul_844" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_671" op: "Assign" input: "bert/encoder/layer_9/attention/output/dense/kernel" input: "sub_156" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_672" op: "Assign" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m" input: "add_528" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_673" op: "Assign" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v" input: "add_529" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/attention/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/output/dense/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/attention/output/dense/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_9/attention/output/dense/bias/adam_m" input: "bert/encoder/layer_9/attention/output/dense/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/attention/output/dense/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_9/attention/output/dense/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/output/dense/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/attention/output/dense/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_9/attention/output/dense/bias/adam_v" input: "bert/encoder/layer_9/attention/output/dense/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/attention/output/dense/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_9/attention/output/dense/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_845/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_845" op: "Mul" input: "Mul_845/x" input: "bert/encoder/layer_9/attention/output/dense/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_846/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_846" op: "Mul" input: "Mul_846/x" input: "clip_by_global_norm/clip_by_global_norm/_156" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_532" op: "Add" input: "Mul_845" input: "Mul_846" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_847/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_847" op: "Mul" input: "Mul_847/x" input: "bert/encoder/layer_9/attention/output/dense/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_156" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_156" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_848/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_848" op: "Mul" input: "Mul_848/x" input: "Square_156" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_533" op: "Add" input: "Mul_847" input: "Mul_848" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_156" op: "Sqrt" input: "add_533" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_534/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_534" op: "Add" input: "Sqrt_156" input: "add_534/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_157" op: "RealDiv" input: "add_532" input: "add_534" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_849" op: "Mul" input: "add_2" input: "truediv_157" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_157" op: "Sub" input: "bert/encoder/layer_9/attention/output/dense/bias/read" input: "mul_849" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_674" op: "Assign" input: "bert/encoder/layer_9/attention/output/dense/bias" input: "sub_157" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_675" op: "Assign" input: "bert/encoder/layer_9/attention/output/dense/bias/adam_m" input: "add_532" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_676" op: "Assign" input: "bert/encoder/layer_9/attention/output/dense/bias/adam_v" input: "add_533" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_m/read" op: "Identity" input: "bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_v/read" op: "Identity" input: "bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_850/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_850" op: "Mul" input: "Mul_850/x" input: "bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_851/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_851" op: "Mul" input: "Mul_851/x" input: "clip_by_global_norm/clip_by_global_norm/_157" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_535" op: "Add" input: "Mul_850" input: "Mul_851" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_852/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_852" op: "Mul" input: "Mul_852/x" input: "bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_157" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_157" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_853/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_853" op: "Mul" input: "Mul_853/x" input: "Square_157" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_536" op: "Add" input: "Mul_852" input: "Mul_853" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_157" op: "Sqrt" input: "add_536" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_537/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_537" op: "Add" input: "Sqrt_157" input: "add_537/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_158" op: "RealDiv" input: "add_535" input: "add_537" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_854" op: "Mul" input: "add_2" input: "truediv_158" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_158" op: "Sub" input: "bert/encoder/layer_9/attention/output/LayerNorm/beta/read" input: "mul_854" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_677" op: "Assign" input: "bert/encoder/layer_9/attention/output/LayerNorm/beta" input: "sub_158" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_678" op: "Assign" input: "bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_m" input: "add_535" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_679" op: "Assign" input: "bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_v" input: "add_536" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m/read" op: "Identity" input: "bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v/read" op: "Identity" input: "bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_855/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_855" op: "Mul" input: "Mul_855/x" input: "bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_856/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_856" op: "Mul" input: "Mul_856/x" input: "clip_by_global_norm/clip_by_global_norm/_158" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_538" op: "Add" input: "Mul_855" input: "Mul_856" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_857/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_857" op: "Mul" input: "Mul_857/x" input: "bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_158" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_158" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_858/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_858" op: "Mul" input: "Mul_858/x" input: "Square_158" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_539" op: "Add" input: "Mul_857" input: "Mul_858" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_158" op: "Sqrt" input: "add_539" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_540/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_540" op: "Add" input: "Sqrt_158" input: "add_540/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_159" op: "RealDiv" input: "add_538" input: "add_540" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_859" op: "Mul" input: "add_2" input: "truediv_159" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_159" op: "Sub" input: "bert/encoder/layer_9/attention/output/LayerNorm/gamma/read" input: "mul_859" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_680" op: "Assign" input: "bert/encoder/layer_9/attention/output/LayerNorm/gamma" input: "sub_159" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_681" op: "Assign" input: "bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m" input: "add_538" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_682" op: "Assign" input: "bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v" input: "add_539" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_860/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_860" op: "Mul" input: "Mul_860/x" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_861/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_861" op: "Mul" input: "Mul_861/x" input: "clip_by_global_norm/clip_by_global_norm/_159" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_541" op: "Add" input: "Mul_860" input: "Mul_861" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_862/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_862" op: "Mul" input: "Mul_862/x" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Square_159" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_159" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_863/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_863" op: "Mul" input: "Mul_863/x" input: "Square_159" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_542" op: "Add" input: "Mul_862" input: "Mul_863" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Sqrt_159" op: "Sqrt" input: "add_542" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_543/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_543" op: "Add" input: "Sqrt_159" input: "add_543/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "truediv_160" op: "RealDiv" input: "add_541" input: "add_543" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "mul_864/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_864" op: "Mul" input: "mul_864/x" input: "bert/encoder/layer_9/intermediate/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_544" op: "Add" input: "truediv_160" input: "mul_864" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "mul_865" op: "Mul" input: "add_2" input: "add_544" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "sub_160" op: "Sub" input: "bert/encoder/layer_9/intermediate/dense/kernel/read" input: "mul_865" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Assign_683" op: "Assign" input: "bert/encoder/layer_9/intermediate/dense/kernel" input: "sub_160" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_684" op: "Assign" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m" input: "add_541" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_685" op: "Assign" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v" input: "add_542" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_9/intermediate/dense/bias/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/intermediate/dense/bias/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_9/intermediate/dense/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/intermediate/dense/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_m" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/intermediate/dense/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_9/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_9/intermediate/dense/bias/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/intermediate/dense/bias/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_9/intermediate/dense/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/intermediate/dense/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_v" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/intermediate/dense/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_866/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_866" op: "Mul" input: "Mul_866/x" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_867/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_867" op: "Mul" input: "Mul_867/x" input: "clip_by_global_norm/clip_by_global_norm/_160" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_545" op: "Add" input: "Mul_866" input: "Mul_867" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_868/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_868" op: "Mul" input: "Mul_868/x" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Square_160" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_160" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_869/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_869" op: "Mul" input: "Mul_869/x" input: "Square_160" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_546" op: "Add" input: "Mul_868" input: "Mul_869" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Sqrt_160" op: "Sqrt" input: "add_546" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_547/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_547" op: "Add" input: "Sqrt_160" input: "add_547/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "truediv_161" op: "RealDiv" input: "add_545" input: "add_547" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "mul_870" op: "Mul" input: "add_2" input: "truediv_161" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "sub_161" op: "Sub" input: "bert/encoder/layer_9/intermediate/dense/bias/read" input: "mul_870" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Assign_686" op: "Assign" input: "bert/encoder/layer_9/intermediate/dense/bias" input: "sub_161" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_687" op: "Assign" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_m" input: "add_545" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_688" op: "Assign" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_v" input: "add_546" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_9/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_9/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_9/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_9/output/dense/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/output/dense/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_9/output/dense/kernel/adam_m" input: "bert/encoder/layer_9/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/output/dense/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_9/output/dense/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_9/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_9/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_9/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_9/output/dense/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/output/dense/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_9/output/dense/kernel/adam_v" input: "bert/encoder/layer_9/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/output/dense/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_9/output/dense/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_871/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_871" op: "Mul" input: "Mul_871/x" input: "bert/encoder/layer_9/output/dense/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_872/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_872" op: "Mul" input: "Mul_872/x" input: "clip_by_global_norm/clip_by_global_norm/_161" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_548" op: "Add" input: "Mul_871" input: "Mul_872" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_873/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_873" op: "Mul" input: "Mul_873/x" input: "bert/encoder/layer_9/output/dense/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Square_161" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_161" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_874/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_874" op: "Mul" input: "Mul_874/x" input: "Square_161" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_549" op: "Add" input: "Mul_873" input: "Mul_874" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Sqrt_161" op: "Sqrt" input: "add_549" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_550/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_550" op: "Add" input: "Sqrt_161" input: "add_550/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "truediv_162" op: "RealDiv" input: "add_548" input: "add_550" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "mul_875/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_875" op: "Mul" input: "mul_875/x" input: "bert/encoder/layer_9/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_551" op: "Add" input: "truediv_162" input: "mul_875" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "mul_876" op: "Mul" input: "add_2" input: "add_551" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "sub_162" op: "Sub" input: "bert/encoder/layer_9/output/dense/kernel/read" input: "mul_876" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Assign_689" op: "Assign" input: "bert/encoder/layer_9/output/dense/kernel" input: "sub_162" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_690" op: "Assign" input: "bert/encoder/layer_9/output/dense/kernel/adam_m" input: "add_548" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_691" op: "Assign" input: "bert/encoder/layer_9/output/dense/kernel/adam_v" input: "add_549" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/output/dense/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/output/dense/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_9/output/dense/bias/adam_m" input: "bert/encoder/layer_9/output/dense/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/output/dense/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_9/output/dense/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/output/dense/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/output/dense/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_9/output/dense/bias/adam_v" input: "bert/encoder/layer_9/output/dense/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/output/dense/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_9/output/dense/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_877/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_877" op: "Mul" input: "Mul_877/x" input: "bert/encoder/layer_9/output/dense/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_878/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_878" op: "Mul" input: "Mul_878/x" input: "clip_by_global_norm/clip_by_global_norm/_162" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_552" op: "Add" input: "Mul_877" input: "Mul_878" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_879/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_879" op: "Mul" input: "Mul_879/x" input: "bert/encoder/layer_9/output/dense/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_162" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_162" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_880/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_880" op: "Mul" input: "Mul_880/x" input: "Square_162" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_553" op: "Add" input: "Mul_879" input: "Mul_880" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_162" op: "Sqrt" input: "add_553" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_554/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_554" op: "Add" input: "Sqrt_162" input: "add_554/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_163" op: "RealDiv" input: "add_552" input: "add_554" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_881" op: "Mul" input: "add_2" input: "truediv_163" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_163" op: "Sub" input: "bert/encoder/layer_9/output/dense/bias/read" input: "mul_881" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_692" op: "Assign" input: "bert/encoder/layer_9/output/dense/bias" input: "sub_163" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_693" op: "Assign" input: "bert/encoder/layer_9/output/dense/bias/adam_m" input: "add_552" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_694" op: "Assign" input: "bert/encoder/layer_9/output/dense/bias/adam_v" input: "add_553" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/output/LayerNorm/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/output/LayerNorm/beta/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/output/LayerNorm/beta/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_9/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_9/output/LayerNorm/beta/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/output/LayerNorm/beta/adam_m/read" op: "Identity" input: "bert/encoder/layer_9/output/LayerNorm/beta/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/output/LayerNorm/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/output/LayerNorm/beta/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/output/LayerNorm/beta/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_9/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_9/output/LayerNorm/beta/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/output/LayerNorm/beta/adam_v/read" op: "Identity" input: "bert/encoder/layer_9/output/LayerNorm/beta/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_882/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_882" op: "Mul" input: "Mul_882/x" input: "bert/encoder/layer_9/output/LayerNorm/beta/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_883/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_883" op: "Mul" input: "Mul_883/x" input: "clip_by_global_norm/clip_by_global_norm/_163" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_555" op: "Add" input: "Mul_882" input: "Mul_883" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_884/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_884" op: "Mul" input: "Mul_884/x" input: "bert/encoder/layer_9/output/LayerNorm/beta/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_163" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_163" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_885/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_885" op: "Mul" input: "Mul_885/x" input: "Square_163" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_556" op: "Add" input: "Mul_884" input: "Mul_885" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_163" op: "Sqrt" input: "add_556" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_557/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_557" op: "Add" input: "Sqrt_163" input: "add_557/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_164" op: "RealDiv" input: "add_555" input: "add_557" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_886" op: "Mul" input: "add_2" input: "truediv_164" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_164" op: "Sub" input: "bert/encoder/layer_9/output/LayerNorm/beta/read" input: "mul_886" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_695" op: "Assign" input: "bert/encoder/layer_9/output/LayerNorm/beta" input: "sub_164" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_696" op: "Assign" input: "bert/encoder/layer_9/output/LayerNorm/beta/adam_m" input: "add_555" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_697" op: "Assign" input: "bert/encoder/layer_9/output/LayerNorm/beta/adam_v" input: "add_556" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/output/LayerNorm/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/output/LayerNorm/gamma/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/output/LayerNorm/gamma/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_9/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_9/output/LayerNorm/gamma/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/output/LayerNorm/gamma/adam_m/read" op: "Identity" input: "bert/encoder/layer_9/output/LayerNorm/gamma/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/output/LayerNorm/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/output/LayerNorm/gamma/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_9/output/LayerNorm/gamma/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_9/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_9/output/LayerNorm/gamma/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_9/output/LayerNorm/gamma/adam_v/read" op: "Identity" input: "bert/encoder/layer_9/output/LayerNorm/gamma/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_887/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_887" op: "Mul" input: "Mul_887/x" input: "bert/encoder/layer_9/output/LayerNorm/gamma/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_888/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_888" op: "Mul" input: "Mul_888/x" input: "clip_by_global_norm/clip_by_global_norm/_164" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_558" op: "Add" input: "Mul_887" input: "Mul_888" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_889/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_889" op: "Mul" input: "Mul_889/x" input: "bert/encoder/layer_9/output/LayerNorm/gamma/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_164" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_164" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_890/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_890" op: "Mul" input: "Mul_890/x" input: "Square_164" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_559" op: "Add" input: "Mul_889" input: "Mul_890" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_164" op: "Sqrt" input: "add_559" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_560/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_560" op: "Add" input: "Sqrt_164" input: "add_560/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_165" op: "RealDiv" input: "add_558" input: "add_560" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_891" op: "Mul" input: "add_2" input: "truediv_165" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_165" op: "Sub" input: "bert/encoder/layer_9/output/LayerNorm/gamma/read" input: "mul_891" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_698" op: "Assign" input: "bert/encoder/layer_9/output/LayerNorm/gamma" input: "sub_165" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_699" op: "Assign" input: "bert/encoder/layer_9/output/LayerNorm/gamma/adam_m" input: "add_558" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_700" op: "Assign" input: "bert/encoder/layer_9/output/LayerNorm/gamma/adam_v" input: "add_559" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_m" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_v" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_892/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_892" op: "Mul" input: "Mul_892/x" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_893/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_893" op: "Mul" input: "Mul_893/x" input: "clip_by_global_norm/clip_by_global_norm/_165" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_561" op: "Add" input: "Mul_892" input: "Mul_893" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_894/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_894" op: "Mul" input: "Mul_894/x" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_165" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_165" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_895/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_895" op: "Mul" input: "Mul_895/x" input: "Square_165" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_562" op: "Add" input: "Mul_894" input: "Mul_895" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_165" op: "Sqrt" input: "add_562" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_563/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_563" op: "Add" input: "Sqrt_165" input: "add_563/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_166" op: "RealDiv" input: "add_561" input: "add_563" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_896/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_896" op: "Mul" input: "mul_896/x" input: "bert/encoder/layer_10/attention/self/query/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_564" op: "Add" input: "truediv_166" input: "mul_896" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_897" op: "Mul" input: "add_2" input: "add_564" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_166" op: "Sub" input: "bert/encoder/layer_10/attention/self/query/kernel/read" input: "mul_897" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_701" op: "Assign" input: "bert/encoder/layer_10/attention/self/query/kernel" input: "sub_166" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_702" op: "Assign" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_m" input: "add_561" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_703" op: "Assign" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_v" input: "add_562" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/attention/self/query/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/self/query/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/attention/self/query/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_10/attention/self/query/bias/adam_m" input: "bert/encoder/layer_10/attention/self/query/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/attention/self/query/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_10/attention/self/query/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/self/query/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/self/query/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/attention/self/query/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_10/attention/self/query/bias/adam_v" input: "bert/encoder/layer_10/attention/self/query/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/attention/self/query/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_10/attention/self/query/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_898/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_898" op: "Mul" input: "Mul_898/x" input: "bert/encoder/layer_10/attention/self/query/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_899/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_899" op: "Mul" input: "Mul_899/x" input: "clip_by_global_norm/clip_by_global_norm/_166" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_565" op: "Add" input: "Mul_898" input: "Mul_899" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_900/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_900" op: "Mul" input: "Mul_900/x" input: "bert/encoder/layer_10/attention/self/query/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_166" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_166" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_901/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_901" op: "Mul" input: "Mul_901/x" input: "Square_166" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_566" op: "Add" input: "Mul_900" input: "Mul_901" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_166" op: "Sqrt" input: "add_566" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_567/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_567" op: "Add" input: "Sqrt_166" input: "add_567/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_167" op: "RealDiv" input: "add_565" input: "add_567" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_902" op: "Mul" input: "add_2" input: "truediv_167" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_167" op: "Sub" input: "bert/encoder/layer_10/attention/self/query/bias/read" input: "mul_902" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_704" op: "Assign" input: "bert/encoder/layer_10/attention/self/query/bias" input: "sub_167" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_705" op: "Assign" input: "bert/encoder/layer_10/attention/self/query/bias/adam_m" input: "add_565" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_706" op: "Assign" input: "bert/encoder/layer_10/attention/self/query/bias/adam_v" input: "add_566" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_m" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_v" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_903/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_903" op: "Mul" input: "Mul_903/x" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_904/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_904" op: "Mul" input: "Mul_904/x" input: "clip_by_global_norm/clip_by_global_norm/_167" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_568" op: "Add" input: "Mul_903" input: "Mul_904" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_905/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_905" op: "Mul" input: "Mul_905/x" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_167" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_167" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_906/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_906" op: "Mul" input: "Mul_906/x" input: "Square_167" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_569" op: "Add" input: "Mul_905" input: "Mul_906" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_167" op: "Sqrt" input: "add_569" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_570/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_570" op: "Add" input: "Sqrt_167" input: "add_570/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_168" op: "RealDiv" input: "add_568" input: "add_570" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_907/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_907" op: "Mul" input: "mul_907/x" input: "bert/encoder/layer_10/attention/self/key/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_571" op: "Add" input: "truediv_168" input: "mul_907" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_908" op: "Mul" input: "add_2" input: "add_571" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_168" op: "Sub" input: "bert/encoder/layer_10/attention/self/key/kernel/read" input: "mul_908" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_707" op: "Assign" input: "bert/encoder/layer_10/attention/self/key/kernel" input: "sub_168" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_708" op: "Assign" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_m" input: "add_568" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_709" op: "Assign" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_v" input: "add_569" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/attention/self/key/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/self/key/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/attention/self/key/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_10/attention/self/key/bias/adam_m" input: "bert/encoder/layer_10/attention/self/key/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/attention/self/key/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_10/attention/self/key/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/self/key/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/self/key/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/attention/self/key/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_10/attention/self/key/bias/adam_v" input: "bert/encoder/layer_10/attention/self/key/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/attention/self/key/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_10/attention/self/key/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_909/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_909" op: "Mul" input: "Mul_909/x" input: "bert/encoder/layer_10/attention/self/key/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_910/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_910" op: "Mul" input: "Mul_910/x" input: "clip_by_global_norm/clip_by_global_norm/_168" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_572" op: "Add" input: "Mul_909" input: "Mul_910" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_911/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_911" op: "Mul" input: "Mul_911/x" input: "bert/encoder/layer_10/attention/self/key/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_168" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_168" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_912/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_912" op: "Mul" input: "Mul_912/x" input: "Square_168" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_573" op: "Add" input: "Mul_911" input: "Mul_912" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_168" op: "Sqrt" input: "add_573" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_574/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_574" op: "Add" input: "Sqrt_168" input: "add_574/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_169" op: "RealDiv" input: "add_572" input: "add_574" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_913" op: "Mul" input: "add_2" input: "truediv_169" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_169" op: "Sub" input: "bert/encoder/layer_10/attention/self/key/bias/read" input: "mul_913" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_710" op: "Assign" input: "bert/encoder/layer_10/attention/self/key/bias" input: "sub_169" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_711" op: "Assign" input: "bert/encoder/layer_10/attention/self/key/bias/adam_m" input: "add_572" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_712" op: "Assign" input: "bert/encoder/layer_10/attention/self/key/bias/adam_v" input: "add_573" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_m" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_v" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_914/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_914" op: "Mul" input: "Mul_914/x" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_915/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_915" op: "Mul" input: "Mul_915/x" input: "clip_by_global_norm/clip_by_global_norm/_169" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_575" op: "Add" input: "Mul_914" input: "Mul_915" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_916/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_916" op: "Mul" input: "Mul_916/x" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_169" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_169" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_917/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_917" op: "Mul" input: "Mul_917/x" input: "Square_169" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_576" op: "Add" input: "Mul_916" input: "Mul_917" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_169" op: "Sqrt" input: "add_576" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_577/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_577" op: "Add" input: "Sqrt_169" input: "add_577/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_170" op: "RealDiv" input: "add_575" input: "add_577" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_918/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_918" op: "Mul" input: "mul_918/x" input: "bert/encoder/layer_10/attention/self/value/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_578" op: "Add" input: "truediv_170" input: "mul_918" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_919" op: "Mul" input: "add_2" input: "add_578" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_170" op: "Sub" input: "bert/encoder/layer_10/attention/self/value/kernel/read" input: "mul_919" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_713" op: "Assign" input: "bert/encoder/layer_10/attention/self/value/kernel" input: "sub_170" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_714" op: "Assign" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_m" input: "add_575" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_715" op: "Assign" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_v" input: "add_576" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/attention/self/value/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/self/value/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/attention/self/value/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_10/attention/self/value/bias/adam_m" input: "bert/encoder/layer_10/attention/self/value/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/attention/self/value/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_10/attention/self/value/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/self/value/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/self/value/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/attention/self/value/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_10/attention/self/value/bias/adam_v" input: "bert/encoder/layer_10/attention/self/value/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/attention/self/value/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_10/attention/self/value/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_920/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_920" op: "Mul" input: "Mul_920/x" input: "bert/encoder/layer_10/attention/self/value/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_921/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_921" op: "Mul" input: "Mul_921/x" input: "clip_by_global_norm/clip_by_global_norm/_170" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_579" op: "Add" input: "Mul_920" input: "Mul_921" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_922/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_922" op: "Mul" input: "Mul_922/x" input: "bert/encoder/layer_10/attention/self/value/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_170" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_170" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_923/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_923" op: "Mul" input: "Mul_923/x" input: "Square_170" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_580" op: "Add" input: "Mul_922" input: "Mul_923" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_170" op: "Sqrt" input: "add_580" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_581/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_581" op: "Add" input: "Sqrt_170" input: "add_581/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_171" op: "RealDiv" input: "add_579" input: "add_581" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_924" op: "Mul" input: "add_2" input: "truediv_171" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_171" op: "Sub" input: "bert/encoder/layer_10/attention/self/value/bias/read" input: "mul_924" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_716" op: "Assign" input: "bert/encoder/layer_10/attention/self/value/bias" input: "sub_171" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_717" op: "Assign" input: "bert/encoder/layer_10/attention/self/value/bias/adam_m" input: "add_579" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_718" op: "Assign" input: "bert/encoder/layer_10/attention/self/value/bias/adam_v" input: "add_580" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_925/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_925" op: "Mul" input: "Mul_925/x" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_926/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_926" op: "Mul" input: "Mul_926/x" input: "clip_by_global_norm/clip_by_global_norm/_171" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_582" op: "Add" input: "Mul_925" input: "Mul_926" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_927/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_927" op: "Mul" input: "Mul_927/x" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_171" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_171" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_928/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_928" op: "Mul" input: "Mul_928/x" input: "Square_171" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_583" op: "Add" input: "Mul_927" input: "Mul_928" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_171" op: "Sqrt" input: "add_583" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_584/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_584" op: "Add" input: "Sqrt_171" input: "add_584/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_172" op: "RealDiv" input: "add_582" input: "add_584" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_929/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_929" op: "Mul" input: "mul_929/x" input: "bert/encoder/layer_10/attention/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_585" op: "Add" input: "truediv_172" input: "mul_929" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_930" op: "Mul" input: "add_2" input: "add_585" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_172" op: "Sub" input: "bert/encoder/layer_10/attention/output/dense/kernel/read" input: "mul_930" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_719" op: "Assign" input: "bert/encoder/layer_10/attention/output/dense/kernel" input: "sub_172" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_720" op: "Assign" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m" input: "add_582" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_721" op: "Assign" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v" input: "add_583" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/attention/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/output/dense/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/attention/output/dense/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_10/attention/output/dense/bias/adam_m" input: "bert/encoder/layer_10/attention/output/dense/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/attention/output/dense/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_10/attention/output/dense/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/output/dense/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/attention/output/dense/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_10/attention/output/dense/bias/adam_v" input: "bert/encoder/layer_10/attention/output/dense/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/attention/output/dense/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_10/attention/output/dense/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_931/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_931" op: "Mul" input: "Mul_931/x" input: "bert/encoder/layer_10/attention/output/dense/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_932/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_932" op: "Mul" input: "Mul_932/x" input: "clip_by_global_norm/clip_by_global_norm/_172" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_586" op: "Add" input: "Mul_931" input: "Mul_932" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_933/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_933" op: "Mul" input: "Mul_933/x" input: "bert/encoder/layer_10/attention/output/dense/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_172" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_172" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_934/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_934" op: "Mul" input: "Mul_934/x" input: "Square_172" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_587" op: "Add" input: "Mul_933" input: "Mul_934" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_172" op: "Sqrt" input: "add_587" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_588/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_588" op: "Add" input: "Sqrt_172" input: "add_588/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_173" op: "RealDiv" input: "add_586" input: "add_588" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_935" op: "Mul" input: "add_2" input: "truediv_173" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_173" op: "Sub" input: "bert/encoder/layer_10/attention/output/dense/bias/read" input: "mul_935" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_722" op: "Assign" input: "bert/encoder/layer_10/attention/output/dense/bias" input: "sub_173" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_723" op: "Assign" input: "bert/encoder/layer_10/attention/output/dense/bias/adam_m" input: "add_586" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_724" op: "Assign" input: "bert/encoder/layer_10/attention/output/dense/bias/adam_v" input: "add_587" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_m/read" op: "Identity" input: "bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_v/read" op: "Identity" input: "bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_936/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_936" op: "Mul" input: "Mul_936/x" input: "bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_937/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_937" op: "Mul" input: "Mul_937/x" input: "clip_by_global_norm/clip_by_global_norm/_173" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_589" op: "Add" input: "Mul_936" input: "Mul_937" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_938/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_938" op: "Mul" input: "Mul_938/x" input: "bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_173" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_173" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_939/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_939" op: "Mul" input: "Mul_939/x" input: "Square_173" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_590" op: "Add" input: "Mul_938" input: "Mul_939" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_173" op: "Sqrt" input: "add_590" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_591/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_591" op: "Add" input: "Sqrt_173" input: "add_591/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_174" op: "RealDiv" input: "add_589" input: "add_591" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_940" op: "Mul" input: "add_2" input: "truediv_174" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_174" op: "Sub" input: "bert/encoder/layer_10/attention/output/LayerNorm/beta/read" input: "mul_940" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_725" op: "Assign" input: "bert/encoder/layer_10/attention/output/LayerNorm/beta" input: "sub_174" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_726" op: "Assign" input: "bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_m" input: "add_589" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_727" op: "Assign" input: "bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_v" input: "add_590" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m/read" op: "Identity" input: "bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v/read" op: "Identity" input: "bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_941/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_941" op: "Mul" input: "Mul_941/x" input: "bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_942/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_942" op: "Mul" input: "Mul_942/x" input: "clip_by_global_norm/clip_by_global_norm/_174" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_592" op: "Add" input: "Mul_941" input: "Mul_942" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_943/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_943" op: "Mul" input: "Mul_943/x" input: "bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_174" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_174" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_944/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_944" op: "Mul" input: "Mul_944/x" input: "Square_174" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_593" op: "Add" input: "Mul_943" input: "Mul_944" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_174" op: "Sqrt" input: "add_593" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_594/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_594" op: "Add" input: "Sqrt_174" input: "add_594/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_175" op: "RealDiv" input: "add_592" input: "add_594" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_945" op: "Mul" input: "add_2" input: "truediv_175" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_175" op: "Sub" input: "bert/encoder/layer_10/attention/output/LayerNorm/gamma/read" input: "mul_945" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_728" op: "Assign" input: "bert/encoder/layer_10/attention/output/LayerNorm/gamma" input: "sub_175" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_729" op: "Assign" input: "bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m" input: "add_592" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_730" op: "Assign" input: "bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v" input: "add_593" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_946/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_946" op: "Mul" input: "Mul_946/x" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_947/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_947" op: "Mul" input: "Mul_947/x" input: "clip_by_global_norm/clip_by_global_norm/_175" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_595" op: "Add" input: "Mul_946" input: "Mul_947" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_948/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_948" op: "Mul" input: "Mul_948/x" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Square_175" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_175" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_949/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_949" op: "Mul" input: "Mul_949/x" input: "Square_175" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_596" op: "Add" input: "Mul_948" input: "Mul_949" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Sqrt_175" op: "Sqrt" input: "add_596" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_597/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_597" op: "Add" input: "Sqrt_175" input: "add_597/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "truediv_176" op: "RealDiv" input: "add_595" input: "add_597" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "mul_950/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_950" op: "Mul" input: "mul_950/x" input: "bert/encoder/layer_10/intermediate/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_598" op: "Add" input: "truediv_176" input: "mul_950" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "mul_951" op: "Mul" input: "add_2" input: "add_598" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "sub_176" op: "Sub" input: "bert/encoder/layer_10/intermediate/dense/kernel/read" input: "mul_951" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Assign_731" op: "Assign" input: "bert/encoder/layer_10/intermediate/dense/kernel" input: "sub_176" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_732" op: "Assign" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m" input: "add_595" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_733" op: "Assign" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v" input: "add_596" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_10/intermediate/dense/bias/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/intermediate/dense/bias/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_10/intermediate/dense/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/intermediate/dense/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_m" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/intermediate/dense/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_10/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_10/intermediate/dense/bias/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/intermediate/dense/bias/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_10/intermediate/dense/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/intermediate/dense/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_v" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/intermediate/dense/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_952/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_952" op: "Mul" input: "Mul_952/x" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_953/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_953" op: "Mul" input: "Mul_953/x" input: "clip_by_global_norm/clip_by_global_norm/_176" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_599" op: "Add" input: "Mul_952" input: "Mul_953" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_954/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_954" op: "Mul" input: "Mul_954/x" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Square_176" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_176" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_955/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_955" op: "Mul" input: "Mul_955/x" input: "Square_176" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_600" op: "Add" input: "Mul_954" input: "Mul_955" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Sqrt_176" op: "Sqrt" input: "add_600" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_601/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_601" op: "Add" input: "Sqrt_176" input: "add_601/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "truediv_177" op: "RealDiv" input: "add_599" input: "add_601" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "mul_956" op: "Mul" input: "add_2" input: "truediv_177" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "sub_177" op: "Sub" input: "bert/encoder/layer_10/intermediate/dense/bias/read" input: "mul_956" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Assign_734" op: "Assign" input: "bert/encoder/layer_10/intermediate/dense/bias" input: "sub_177" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_735" op: "Assign" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_m" input: "add_599" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_736" op: "Assign" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_v" input: "add_600" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_10/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_10/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_10/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_10/output/dense/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/output/dense/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_10/output/dense/kernel/adam_m" input: "bert/encoder/layer_10/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/output/dense/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_10/output/dense/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_10/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_10/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_10/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_10/output/dense/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/output/dense/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_10/output/dense/kernel/adam_v" input: "bert/encoder/layer_10/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/output/dense/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_10/output/dense/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_957/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_957" op: "Mul" input: "Mul_957/x" input: "bert/encoder/layer_10/output/dense/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_958/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_958" op: "Mul" input: "Mul_958/x" input: "clip_by_global_norm/clip_by_global_norm/_177" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_602" op: "Add" input: "Mul_957" input: "Mul_958" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_959/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_959" op: "Mul" input: "Mul_959/x" input: "bert/encoder/layer_10/output/dense/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Square_177" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_177" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_960/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_960" op: "Mul" input: "Mul_960/x" input: "Square_177" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_603" op: "Add" input: "Mul_959" input: "Mul_960" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Sqrt_177" op: "Sqrt" input: "add_603" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_604/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_604" op: "Add" input: "Sqrt_177" input: "add_604/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "truediv_178" op: "RealDiv" input: "add_602" input: "add_604" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "mul_961/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_961" op: "Mul" input: "mul_961/x" input: "bert/encoder/layer_10/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_605" op: "Add" input: "truediv_178" input: "mul_961" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "mul_962" op: "Mul" input: "add_2" input: "add_605" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "sub_178" op: "Sub" input: "bert/encoder/layer_10/output/dense/kernel/read" input: "mul_962" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Assign_737" op: "Assign" input: "bert/encoder/layer_10/output/dense/kernel" input: "sub_178" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_738" op: "Assign" input: "bert/encoder/layer_10/output/dense/kernel/adam_m" input: "add_602" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_739" op: "Assign" input: "bert/encoder/layer_10/output/dense/kernel/adam_v" input: "add_603" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/output/dense/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/output/dense/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_10/output/dense/bias/adam_m" input: "bert/encoder/layer_10/output/dense/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/output/dense/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_10/output/dense/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/output/dense/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/output/dense/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_10/output/dense/bias/adam_v" input: "bert/encoder/layer_10/output/dense/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/output/dense/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_10/output/dense/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_963/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_963" op: "Mul" input: "Mul_963/x" input: "bert/encoder/layer_10/output/dense/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_964/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_964" op: "Mul" input: "Mul_964/x" input: "clip_by_global_norm/clip_by_global_norm/_178" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_606" op: "Add" input: "Mul_963" input: "Mul_964" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_965/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_965" op: "Mul" input: "Mul_965/x" input: "bert/encoder/layer_10/output/dense/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_178" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_178" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_966/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_966" op: "Mul" input: "Mul_966/x" input: "Square_178" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_607" op: "Add" input: "Mul_965" input: "Mul_966" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_178" op: "Sqrt" input: "add_607" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_608/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_608" op: "Add" input: "Sqrt_178" input: "add_608/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_179" op: "RealDiv" input: "add_606" input: "add_608" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_967" op: "Mul" input: "add_2" input: "truediv_179" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_179" op: "Sub" input: "bert/encoder/layer_10/output/dense/bias/read" input: "mul_967" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_740" op: "Assign" input: "bert/encoder/layer_10/output/dense/bias" input: "sub_179" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_741" op: "Assign" input: "bert/encoder/layer_10/output/dense/bias/adam_m" input: "add_606" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_742" op: "Assign" input: "bert/encoder/layer_10/output/dense/bias/adam_v" input: "add_607" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/output/LayerNorm/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/output/LayerNorm/beta/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/output/LayerNorm/beta/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_10/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_10/output/LayerNorm/beta/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/output/LayerNorm/beta/adam_m/read" op: "Identity" input: "bert/encoder/layer_10/output/LayerNorm/beta/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/output/LayerNorm/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/output/LayerNorm/beta/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/output/LayerNorm/beta/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_10/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_10/output/LayerNorm/beta/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/output/LayerNorm/beta/adam_v/read" op: "Identity" input: "bert/encoder/layer_10/output/LayerNorm/beta/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_968/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_968" op: "Mul" input: "Mul_968/x" input: "bert/encoder/layer_10/output/LayerNorm/beta/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_969/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_969" op: "Mul" input: "Mul_969/x" input: "clip_by_global_norm/clip_by_global_norm/_179" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_609" op: "Add" input: "Mul_968" input: "Mul_969" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_970/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_970" op: "Mul" input: "Mul_970/x" input: "bert/encoder/layer_10/output/LayerNorm/beta/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_179" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_179" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_971/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_971" op: "Mul" input: "Mul_971/x" input: "Square_179" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_610" op: "Add" input: "Mul_970" input: "Mul_971" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_179" op: "Sqrt" input: "add_610" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_611/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_611" op: "Add" input: "Sqrt_179" input: "add_611/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_180" op: "RealDiv" input: "add_609" input: "add_611" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_972" op: "Mul" input: "add_2" input: "truediv_180" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_180" op: "Sub" input: "bert/encoder/layer_10/output/LayerNorm/beta/read" input: "mul_972" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_743" op: "Assign" input: "bert/encoder/layer_10/output/LayerNorm/beta" input: "sub_180" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_744" op: "Assign" input: "bert/encoder/layer_10/output/LayerNorm/beta/adam_m" input: "add_609" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_745" op: "Assign" input: "bert/encoder/layer_10/output/LayerNorm/beta/adam_v" input: "add_610" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/output/LayerNorm/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/output/LayerNorm/gamma/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/output/LayerNorm/gamma/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_10/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_10/output/LayerNorm/gamma/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/output/LayerNorm/gamma/adam_m/read" op: "Identity" input: "bert/encoder/layer_10/output/LayerNorm/gamma/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/output/LayerNorm/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/output/LayerNorm/gamma/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_10/output/LayerNorm/gamma/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_10/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_10/output/LayerNorm/gamma/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_10/output/LayerNorm/gamma/adam_v/read" op: "Identity" input: "bert/encoder/layer_10/output/LayerNorm/gamma/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_973/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_973" op: "Mul" input: "Mul_973/x" input: "bert/encoder/layer_10/output/LayerNorm/gamma/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_974/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_974" op: "Mul" input: "Mul_974/x" input: "clip_by_global_norm/clip_by_global_norm/_180" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_612" op: "Add" input: "Mul_973" input: "Mul_974" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_975/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_975" op: "Mul" input: "Mul_975/x" input: "bert/encoder/layer_10/output/LayerNorm/gamma/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_180" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_180" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_976/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_976" op: "Mul" input: "Mul_976/x" input: "Square_180" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_613" op: "Add" input: "Mul_975" input: "Mul_976" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_180" op: "Sqrt" input: "add_613" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_614/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_614" op: "Add" input: "Sqrt_180" input: "add_614/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_181" op: "RealDiv" input: "add_612" input: "add_614" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_977" op: "Mul" input: "add_2" input: "truediv_181" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_181" op: "Sub" input: "bert/encoder/layer_10/output/LayerNorm/gamma/read" input: "mul_977" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_746" op: "Assign" input: "bert/encoder/layer_10/output/LayerNorm/gamma" input: "sub_181" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_747" op: "Assign" input: "bert/encoder/layer_10/output/LayerNorm/gamma/adam_m" input: "add_612" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_748" op: "Assign" input: "bert/encoder/layer_10/output/LayerNorm/gamma/adam_v" input: "add_613" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_m" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_v" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_978/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_978" op: "Mul" input: "Mul_978/x" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_979/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_979" op: "Mul" input: "Mul_979/x" input: "clip_by_global_norm/clip_by_global_norm/_181" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_615" op: "Add" input: "Mul_978" input: "Mul_979" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_980/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_980" op: "Mul" input: "Mul_980/x" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_181" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_181" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_981/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_981" op: "Mul" input: "Mul_981/x" input: "Square_181" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_616" op: "Add" input: "Mul_980" input: "Mul_981" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_181" op: "Sqrt" input: "add_616" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_617/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_617" op: "Add" input: "Sqrt_181" input: "add_617/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_182" op: "RealDiv" input: "add_615" input: "add_617" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_982/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_982" op: "Mul" input: "mul_982/x" input: "bert/encoder/layer_11/attention/self/query/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_618" op: "Add" input: "truediv_182" input: "mul_982" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_983" op: "Mul" input: "add_2" input: "add_618" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_182" op: "Sub" input: "bert/encoder/layer_11/attention/self/query/kernel/read" input: "mul_983" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_749" op: "Assign" input: "bert/encoder/layer_11/attention/self/query/kernel" input: "sub_182" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_750" op: "Assign" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_m" input: "add_615" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_751" op: "Assign" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_v" input: "add_616" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/attention/self/query/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/self/query/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/attention/self/query/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_11/attention/self/query/bias/adam_m" input: "bert/encoder/layer_11/attention/self/query/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/attention/self/query/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_11/attention/self/query/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/self/query/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/self/query/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/attention/self/query/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_11/attention/self/query/bias/adam_v" input: "bert/encoder/layer_11/attention/self/query/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/attention/self/query/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_11/attention/self/query/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_984/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_984" op: "Mul" input: "Mul_984/x" input: "bert/encoder/layer_11/attention/self/query/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_985/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_985" op: "Mul" input: "Mul_985/x" input: "clip_by_global_norm/clip_by_global_norm/_182" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_619" op: "Add" input: "Mul_984" input: "Mul_985" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_986/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_986" op: "Mul" input: "Mul_986/x" input: "bert/encoder/layer_11/attention/self/query/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_182" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_182" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_987/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_987" op: "Mul" input: "Mul_987/x" input: "Square_182" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_620" op: "Add" input: "Mul_986" input: "Mul_987" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_182" op: "Sqrt" input: "add_620" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_621/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_621" op: "Add" input: "Sqrt_182" input: "add_621/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_183" op: "RealDiv" input: "add_619" input: "add_621" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_988" op: "Mul" input: "add_2" input: "truediv_183" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_183" op: "Sub" input: "bert/encoder/layer_11/attention/self/query/bias/read" input: "mul_988" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_752" op: "Assign" input: "bert/encoder/layer_11/attention/self/query/bias" input: "sub_183" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_753" op: "Assign" input: "bert/encoder/layer_11/attention/self/query/bias/adam_m" input: "add_619" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_754" op: "Assign" input: "bert/encoder/layer_11/attention/self/query/bias/adam_v" input: "add_620" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_m" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_v" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_989/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_989" op: "Mul" input: "Mul_989/x" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_990/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_990" op: "Mul" input: "Mul_990/x" input: "clip_by_global_norm/clip_by_global_norm/_183" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_622" op: "Add" input: "Mul_989" input: "Mul_990" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_991/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_991" op: "Mul" input: "Mul_991/x" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_183" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_183" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_992/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_992" op: "Mul" input: "Mul_992/x" input: "Square_183" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_623" op: "Add" input: "Mul_991" input: "Mul_992" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_183" op: "Sqrt" input: "add_623" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_624/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_624" op: "Add" input: "Sqrt_183" input: "add_624/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_184" op: "RealDiv" input: "add_622" input: "add_624" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_993/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_993" op: "Mul" input: "mul_993/x" input: "bert/encoder/layer_11/attention/self/key/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_625" op: "Add" input: "truediv_184" input: "mul_993" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_994" op: "Mul" input: "add_2" input: "add_625" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_184" op: "Sub" input: "bert/encoder/layer_11/attention/self/key/kernel/read" input: "mul_994" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_755" op: "Assign" input: "bert/encoder/layer_11/attention/self/key/kernel" input: "sub_184" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_756" op: "Assign" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_m" input: "add_622" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_757" op: "Assign" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_v" input: "add_623" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/attention/self/key/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/self/key/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/attention/self/key/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_11/attention/self/key/bias/adam_m" input: "bert/encoder/layer_11/attention/self/key/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/attention/self/key/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_11/attention/self/key/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/self/key/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/self/key/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/attention/self/key/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_11/attention/self/key/bias/adam_v" input: "bert/encoder/layer_11/attention/self/key/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/attention/self/key/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_11/attention/self/key/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_995/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_995" op: "Mul" input: "Mul_995/x" input: "bert/encoder/layer_11/attention/self/key/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_996/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_996" op: "Mul" input: "Mul_996/x" input: "clip_by_global_norm/clip_by_global_norm/_184" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_626" op: "Add" input: "Mul_995" input: "Mul_996" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_997/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_997" op: "Mul" input: "Mul_997/x" input: "bert/encoder/layer_11/attention/self/key/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_184" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_184" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_998/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_998" op: "Mul" input: "Mul_998/x" input: "Square_184" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_627" op: "Add" input: "Mul_997" input: "Mul_998" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_184" op: "Sqrt" input: "add_627" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_628/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_628" op: "Add" input: "Sqrt_184" input: "add_628/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_185" op: "RealDiv" input: "add_626" input: "add_628" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_999" op: "Mul" input: "add_2" input: "truediv_185" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_185" op: "Sub" input: "bert/encoder/layer_11/attention/self/key/bias/read" input: "mul_999" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_758" op: "Assign" input: "bert/encoder/layer_11/attention/self/key/bias" input: "sub_185" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_759" op: "Assign" input: "bert/encoder/layer_11/attention/self/key/bias/adam_m" input: "add_626" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_760" op: "Assign" input: "bert/encoder/layer_11/attention/self/key/bias/adam_v" input: "add_627" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_m" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_v" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_1000/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_1000" op: "Mul" input: "Mul_1000/x" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_1001/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_1001" op: "Mul" input: "Mul_1001/x" input: "clip_by_global_norm/clip_by_global_norm/_185" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_629" op: "Add" input: "Mul_1000" input: "Mul_1001" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_1002/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_1002" op: "Mul" input: "Mul_1002/x" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_185" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_185" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_1003/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_1003" op: "Mul" input: "Mul_1003/x" input: "Square_185" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_630" op: "Add" input: "Mul_1002" input: "Mul_1003" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_185" op: "Sqrt" input: "add_630" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_631/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_631" op: "Add" input: "Sqrt_185" input: "add_631/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_186" op: "RealDiv" input: "add_629" input: "add_631" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_1004/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_1004" op: "Mul" input: "mul_1004/x" input: "bert/encoder/layer_11/attention/self/value/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_632" op: "Add" input: "truediv_186" input: "mul_1004" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_1005" op: "Mul" input: "add_2" input: "add_632" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_186" op: "Sub" input: "bert/encoder/layer_11/attention/self/value/kernel/read" input: "mul_1005" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_761" op: "Assign" input: "bert/encoder/layer_11/attention/self/value/kernel" input: "sub_186" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_762" op: "Assign" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_m" input: "add_629" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_763" op: "Assign" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_v" input: "add_630" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/attention/self/value/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/self/value/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/attention/self/value/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_11/attention/self/value/bias/adam_m" input: "bert/encoder/layer_11/attention/self/value/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/attention/self/value/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_11/attention/self/value/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/self/value/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/self/value/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/attention/self/value/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_11/attention/self/value/bias/adam_v" input: "bert/encoder/layer_11/attention/self/value/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/attention/self/value/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_11/attention/self/value/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1006/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_1006" op: "Mul" input: "Mul_1006/x" input: "bert/encoder/layer_11/attention/self/value/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1007/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_1007" op: "Mul" input: "Mul_1007/x" input: "clip_by_global_norm/clip_by_global_norm/_186" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_633" op: "Add" input: "Mul_1006" input: "Mul_1007" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1008/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_1008" op: "Mul" input: "Mul_1008/x" input: "bert/encoder/layer_11/attention/self/value/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_186" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_186" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1009/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_1009" op: "Mul" input: "Mul_1009/x" input: "Square_186" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_634" op: "Add" input: "Mul_1008" input: "Mul_1009" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_186" op: "Sqrt" input: "add_634" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_635/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_635" op: "Add" input: "Sqrt_186" input: "add_635/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_187" op: "RealDiv" input: "add_633" input: "add_635" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_1010" op: "Mul" input: "add_2" input: "truediv_187" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_187" op: "Sub" input: "bert/encoder/layer_11/attention/self/value/bias/read" input: "mul_1010" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_764" op: "Assign" input: "bert/encoder/layer_11/attention/self/value/bias" input: "sub_187" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_765" op: "Assign" input: "bert/encoder/layer_11/attention/self/value/bias/adam_m" input: "add_633" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_766" op: "Assign" input: "bert/encoder/layer_11/attention/self/value/bias/adam_v" input: "add_634" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_1011/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_1011" op: "Mul" input: "Mul_1011/x" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_1012/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_1012" op: "Mul" input: "Mul_1012/x" input: "clip_by_global_norm/clip_by_global_norm/_187" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_636" op: "Add" input: "Mul_1011" input: "Mul_1012" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_1013/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_1013" op: "Mul" input: "Mul_1013/x" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_187" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_187" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_1014/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_1014" op: "Mul" input: "Mul_1014/x" input: "Square_187" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_637" op: "Add" input: "Mul_1013" input: "Mul_1014" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_187" op: "Sqrt" input: "add_637" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_638/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_638" op: "Add" input: "Sqrt_187" input: "add_638/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_188" op: "RealDiv" input: "add_636" input: "add_638" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_1015/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_1015" op: "Mul" input: "mul_1015/x" input: "bert/encoder/layer_11/attention/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_639" op: "Add" input: "truediv_188" input: "mul_1015" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_1016" op: "Mul" input: "add_2" input: "add_639" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_188" op: "Sub" input: "bert/encoder/layer_11/attention/output/dense/kernel/read" input: "mul_1016" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_767" op: "Assign" input: "bert/encoder/layer_11/attention/output/dense/kernel" input: "sub_188" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_768" op: "Assign" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m" input: "add_636" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_769" op: "Assign" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v" input: "add_637" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/attention/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/output/dense/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/attention/output/dense/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_11/attention/output/dense/bias/adam_m" input: "bert/encoder/layer_11/attention/output/dense/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/attention/output/dense/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_11/attention/output/dense/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/output/dense/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/attention/output/dense/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_11/attention/output/dense/bias/adam_v" input: "bert/encoder/layer_11/attention/output/dense/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/attention/output/dense/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_11/attention/output/dense/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1017/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_1017" op: "Mul" input: "Mul_1017/x" input: "bert/encoder/layer_11/attention/output/dense/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1018/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_1018" op: "Mul" input: "Mul_1018/x" input: "clip_by_global_norm/clip_by_global_norm/_188" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_640" op: "Add" input: "Mul_1017" input: "Mul_1018" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1019/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_1019" op: "Mul" input: "Mul_1019/x" input: "bert/encoder/layer_11/attention/output/dense/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_188" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_188" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1020/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_1020" op: "Mul" input: "Mul_1020/x" input: "Square_188" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_641" op: "Add" input: "Mul_1019" input: "Mul_1020" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_188" op: "Sqrt" input: "add_641" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_642/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_642" op: "Add" input: "Sqrt_188" input: "add_642/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_189" op: "RealDiv" input: "add_640" input: "add_642" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_1021" op: "Mul" input: "add_2" input: "truediv_189" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_189" op: "Sub" input: "bert/encoder/layer_11/attention/output/dense/bias/read" input: "mul_1021" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_770" op: "Assign" input: "bert/encoder/layer_11/attention/output/dense/bias" input: "sub_189" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_771" op: "Assign" input: "bert/encoder/layer_11/attention/output/dense/bias/adam_m" input: "add_640" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_772" op: "Assign" input: "bert/encoder/layer_11/attention/output/dense/bias/adam_v" input: "add_641" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_m/read" op: "Identity" input: "bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_v/read" op: "Identity" input: "bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1022/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_1022" op: "Mul" input: "Mul_1022/x" input: "bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1023/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_1023" op: "Mul" input: "Mul_1023/x" input: "clip_by_global_norm/clip_by_global_norm/_189" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_643" op: "Add" input: "Mul_1022" input: "Mul_1023" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1024/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_1024" op: "Mul" input: "Mul_1024/x" input: "bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_189" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_189" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1025/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_1025" op: "Mul" input: "Mul_1025/x" input: "Square_189" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_644" op: "Add" input: "Mul_1024" input: "Mul_1025" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_189" op: "Sqrt" input: "add_644" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_645/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_645" op: "Add" input: "Sqrt_189" input: "add_645/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_190" op: "RealDiv" input: "add_643" input: "add_645" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_1026" op: "Mul" input: "add_2" input: "truediv_190" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_190" op: "Sub" input: "bert/encoder/layer_11/attention/output/LayerNorm/beta/read" input: "mul_1026" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_773" op: "Assign" input: "bert/encoder/layer_11/attention/output/LayerNorm/beta" input: "sub_190" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_774" op: "Assign" input: "bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_m" input: "add_643" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_775" op: "Assign" input: "bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_v" input: "add_644" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m/read" op: "Identity" input: "bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v/read" op: "Identity" input: "bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1027/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_1027" op: "Mul" input: "Mul_1027/x" input: "bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1028/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_1028" op: "Mul" input: "Mul_1028/x" input: "clip_by_global_norm/clip_by_global_norm/_190" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_646" op: "Add" input: "Mul_1027" input: "Mul_1028" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1029/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_1029" op: "Mul" input: "Mul_1029/x" input: "bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_190" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_190" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1030/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_1030" op: "Mul" input: "Mul_1030/x" input: "Square_190" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_647" op: "Add" input: "Mul_1029" input: "Mul_1030" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_190" op: "Sqrt" input: "add_647" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_648/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_648" op: "Add" input: "Sqrt_190" input: "add_648/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_191" op: "RealDiv" input: "add_646" input: "add_648" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_1031" op: "Mul" input: "add_2" input: "truediv_191" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_191" op: "Sub" input: "bert/encoder/layer_11/attention/output/LayerNorm/gamma/read" input: "mul_1031" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_776" op: "Assign" input: "bert/encoder/layer_11/attention/output/LayerNorm/gamma" input: "sub_191" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_777" op: "Assign" input: "bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m" input: "add_646" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_778" op: "Assign" input: "bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v" input: "add_647" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_1032/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_1032" op: "Mul" input: "Mul_1032/x" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_1033/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_1033" op: "Mul" input: "Mul_1033/x" input: "clip_by_global_norm/clip_by_global_norm/_191" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_649" op: "Add" input: "Mul_1032" input: "Mul_1033" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_1034/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_1034" op: "Mul" input: "Mul_1034/x" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Square_191" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_191" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_1035/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_1035" op: "Mul" input: "Mul_1035/x" input: "Square_191" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_650" op: "Add" input: "Mul_1034" input: "Mul_1035" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Sqrt_191" op: "Sqrt" input: "add_650" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_651/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_651" op: "Add" input: "Sqrt_191" input: "add_651/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "truediv_192" op: "RealDiv" input: "add_649" input: "add_651" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "mul_1036/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_1036" op: "Mul" input: "mul_1036/x" input: "bert/encoder/layer_11/intermediate/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_652" op: "Add" input: "truediv_192" input: "mul_1036" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "mul_1037" op: "Mul" input: "add_2" input: "add_652" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "sub_192" op: "Sub" input: "bert/encoder/layer_11/intermediate/dense/kernel/read" input: "mul_1037" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Assign_779" op: "Assign" input: "bert/encoder/layer_11/intermediate/dense/kernel" input: "sub_192" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_780" op: "Assign" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m" input: "add_649" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_781" op: "Assign" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v" input: "add_650" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_11/intermediate/dense/bias/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/intermediate/dense/bias/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_11/intermediate/dense/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/intermediate/dense/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_m" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/intermediate/dense/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_11/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_11/intermediate/dense/bias/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/intermediate/dense/bias/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_11/intermediate/dense/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/intermediate/dense/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_v" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/intermediate/dense/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_1038/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_1038" op: "Mul" input: "Mul_1038/x" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_1039/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_1039" op: "Mul" input: "Mul_1039/x" input: "clip_by_global_norm/clip_by_global_norm/_192" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_653" op: "Add" input: "Mul_1038" input: "Mul_1039" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_1040/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_1040" op: "Mul" input: "Mul_1040/x" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Square_192" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_192" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_1041/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_1041" op: "Mul" input: "Mul_1041/x" input: "Square_192" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_654" op: "Add" input: "Mul_1040" input: "Mul_1041" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Sqrt_192" op: "Sqrt" input: "add_654" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_655/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_655" op: "Add" input: "Sqrt_192" input: "add_655/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "truediv_193" op: "RealDiv" input: "add_653" input: "add_655" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "mul_1042" op: "Mul" input: "add_2" input: "truediv_193" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "sub_193" op: "Sub" input: "bert/encoder/layer_11/intermediate/dense/bias/read" input: "mul_1042" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Assign_782" op: "Assign" input: "bert/encoder/layer_11/intermediate/dense/bias" input: "sub_193" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_783" op: "Assign" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_m" input: "add_653" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_784" op: "Assign" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_v" input: "add_654" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_11/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_11/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_11/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_11/output/dense/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/output/dense/kernel/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_11/output/dense/kernel/adam_m" input: "bert/encoder/layer_11/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/output/dense/kernel/adam_m/read" op: "Identity" input: "bert/encoder/layer_11/output/dense/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_11/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_11/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_11/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_11/output/dense/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/output/dense/kernel/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_11/output/dense/kernel/adam_v" input: "bert/encoder/layer_11/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/output/dense/kernel/adam_v/read" op: "Identity" input: "bert/encoder/layer_11/output/dense/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_1043/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_1043" op: "Mul" input: "Mul_1043/x" input: "bert/encoder/layer_11/output/dense/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_1044/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_1044" op: "Mul" input: "Mul_1044/x" input: "clip_by_global_norm/clip_by_global_norm/_193" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_656" op: "Add" input: "Mul_1043" input: "Mul_1044" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_1045/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_1045" op: "Mul" input: "Mul_1045/x" input: "bert/encoder/layer_11/output/dense/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Square_193" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_193" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_1046/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_1046" op: "Mul" input: "Mul_1046/x" input: "Square_193" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_657" op: "Add" input: "Mul_1045" input: "Mul_1046" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Sqrt_193" op: "Sqrt" input: "add_657" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_658/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_658" op: "Add" input: "Sqrt_193" input: "add_658/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "truediv_194" op: "RealDiv" input: "add_656" input: "add_658" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "mul_1047/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_1047" op: "Mul" input: "mul_1047/x" input: "bert/encoder/layer_11/output/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_659" op: "Add" input: "truediv_194" input: "mul_1047" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "mul_1048" op: "Mul" input: "add_2" input: "add_659" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "sub_194" op: "Sub" input: "bert/encoder/layer_11/output/dense/kernel/read" input: "mul_1048" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Assign_785" op: "Assign" input: "bert/encoder/layer_11/output/dense/kernel" input: "sub_194" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_786" op: "Assign" input: "bert/encoder/layer_11/output/dense/kernel/adam_m" input: "add_656" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_787" op: "Assign" input: "bert/encoder/layer_11/output/dense/kernel/adam_v" input: "add_657" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/output/dense/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/output/dense/bias/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_11/output/dense/bias/adam_m" input: "bert/encoder/layer_11/output/dense/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/output/dense/bias/adam_m/read" op: "Identity" input: "bert/encoder/layer_11/output/dense/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/output/dense/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/output/dense/bias/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_11/output/dense/bias/adam_v" input: "bert/encoder/layer_11/output/dense/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/output/dense/bias/adam_v/read" op: "Identity" input: "bert/encoder/layer_11/output/dense/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1049/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_1049" op: "Mul" input: "Mul_1049/x" input: "bert/encoder/layer_11/output/dense/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1050/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_1050" op: "Mul" input: "Mul_1050/x" input: "clip_by_global_norm/clip_by_global_norm/_194" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_660" op: "Add" input: "Mul_1049" input: "Mul_1050" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1051/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_1051" op: "Mul" input: "Mul_1051/x" input: "bert/encoder/layer_11/output/dense/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_194" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_194" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1052/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_1052" op: "Mul" input: "Mul_1052/x" input: "Square_194" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_661" op: "Add" input: "Mul_1051" input: "Mul_1052" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_194" op: "Sqrt" input: "add_661" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_662/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_662" op: "Add" input: "Sqrt_194" input: "add_662/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_195" op: "RealDiv" input: "add_660" input: "add_662" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_1053" op: "Mul" input: "add_2" input: "truediv_195" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_195" op: "Sub" input: "bert/encoder/layer_11/output/dense/bias/read" input: "mul_1053" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_788" op: "Assign" input: "bert/encoder/layer_11/output/dense/bias" input: "sub_195" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_789" op: "Assign" input: "bert/encoder/layer_11/output/dense/bias/adam_m" input: "add_660" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_790" op: "Assign" input: "bert/encoder/layer_11/output/dense/bias/adam_v" input: "add_661" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/output/LayerNorm/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/output/LayerNorm/beta/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/output/LayerNorm/beta/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_11/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_11/output/LayerNorm/beta/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/output/LayerNorm/beta/adam_m/read" op: "Identity" input: "bert/encoder/layer_11/output/LayerNorm/beta/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/output/LayerNorm/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/output/LayerNorm/beta/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/output/LayerNorm/beta/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_11/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_11/output/LayerNorm/beta/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/output/LayerNorm/beta/adam_v/read" op: "Identity" input: "bert/encoder/layer_11/output/LayerNorm/beta/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1054/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_1054" op: "Mul" input: "Mul_1054/x" input: "bert/encoder/layer_11/output/LayerNorm/beta/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1055/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_1055" op: "Mul" input: "Mul_1055/x" input: "clip_by_global_norm/clip_by_global_norm/_195" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_663" op: "Add" input: "Mul_1054" input: "Mul_1055" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1056/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_1056" op: "Mul" input: "Mul_1056/x" input: "bert/encoder/layer_11/output/LayerNorm/beta/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_195" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_195" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1057/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_1057" op: "Mul" input: "Mul_1057/x" input: "Square_195" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_664" op: "Add" input: "Mul_1056" input: "Mul_1057" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_195" op: "Sqrt" input: "add_664" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_665/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_665" op: "Add" input: "Sqrt_195" input: "add_665/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_196" op: "RealDiv" input: "add_663" input: "add_665" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_1058" op: "Mul" input: "add_2" input: "truediv_196" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_196" op: "Sub" input: "bert/encoder/layer_11/output/LayerNorm/beta/read" input: "mul_1058" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_791" op: "Assign" input: "bert/encoder/layer_11/output/LayerNorm/beta" input: "sub_196" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_792" op: "Assign" input: "bert/encoder/layer_11/output/LayerNorm/beta/adam_m" input: "add_663" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_793" op: "Assign" input: "bert/encoder/layer_11/output/LayerNorm/beta/adam_v" input: "add_664" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/output/LayerNorm/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/output/LayerNorm/gamma/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/output/LayerNorm/gamma/adam_m/Assign" op: "Assign" input: "bert/encoder/layer_11/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_11/output/LayerNorm/gamma/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/output/LayerNorm/gamma/adam_m/read" op: "Identity" input: "bert/encoder/layer_11/output/LayerNorm/gamma/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/output/LayerNorm/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/output/LayerNorm/gamma/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/encoder/layer_11/output/LayerNorm/gamma/adam_v/Assign" op: "Assign" input: "bert/encoder/layer_11/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_11/output/LayerNorm/gamma/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/encoder/layer_11/output/LayerNorm/gamma/adam_v/read" op: "Identity" input: "bert/encoder/layer_11/output/LayerNorm/gamma/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1059/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_1059" op: "Mul" input: "Mul_1059/x" input: "bert/encoder/layer_11/output/LayerNorm/gamma/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1060/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_1060" op: "Mul" input: "Mul_1060/x" input: "clip_by_global_norm/clip_by_global_norm/_196" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_666" op: "Add" input: "Mul_1059" input: "Mul_1060" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1061/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_1061" op: "Mul" input: "Mul_1061/x" input: "bert/encoder/layer_11/output/LayerNorm/gamma/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_196" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_196" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1062/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_1062" op: "Mul" input: "Mul_1062/x" input: "Square_196" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_667" op: "Add" input: "Mul_1061" input: "Mul_1062" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_196" op: "Sqrt" input: "add_667" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_668/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_668" op: "Add" input: "Sqrt_196" input: "add_668/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_197" op: "RealDiv" input: "add_666" input: "add_668" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_1063" op: "Mul" input: "add_2" input: "truediv_197" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_197" op: "Sub" input: "bert/encoder/layer_11/output/LayerNorm/gamma/read" input: "mul_1063" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_794" op: "Assign" input: "bert/encoder/layer_11/output/LayerNorm/gamma" input: "sub_197" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_795" op: "Assign" input: "bert/encoder/layer_11/output/LayerNorm/gamma/adam_m" input: "add_666" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_796" op: "Assign" input: "bert/encoder/layer_11/output/LayerNorm/gamma/adam_v" input: "add_667" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/pooler/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/pooler/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/pooler/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/pooler/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/pooler/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/pooler/dense/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/pooler/dense/kernel/adam_m/Assign" op: "Assign" input: "bert/pooler/dense/kernel/adam_m" input: "bert/pooler/dense/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/pooler/dense/kernel/adam_m/read" op: "Identity" input: "bert/pooler/dense/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/pooler/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/pooler/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/pooler/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/pooler/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/pooler/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/pooler/dense/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/pooler/dense/kernel/adam_v/Assign" op: "Assign" input: "bert/pooler/dense/kernel/adam_v" input: "bert/pooler/dense/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/pooler/dense/kernel/adam_v/read" op: "Identity" input: "bert/pooler/dense/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_1064/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_1064" op: "Mul" input: "Mul_1064/x" input: "bert/pooler/dense/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_1065/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_1065" op: "Mul" input: "Mul_1065/x" input: "clip_by_global_norm/clip_by_global_norm/_197" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_669" op: "Add" input: "Mul_1064" input: "Mul_1065" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_1066/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_1066" op: "Mul" input: "Mul_1066/x" input: "bert/pooler/dense/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_197" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_197" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_1067/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_1067" op: "Mul" input: "Mul_1067/x" input: "Square_197" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_670" op: "Add" input: "Mul_1066" input: "Mul_1067" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_197" op: "Sqrt" input: "add_670" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_671/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_671" op: "Add" input: "Sqrt_197" input: "add_671/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_198" op: "RealDiv" input: "add_669" input: "add_671" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_1068/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_1068" op: "Mul" input: "mul_1068/x" input: "bert/pooler/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_672" op: "Add" input: "truediv_198" input: "mul_1068" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_1069" op: "Mul" input: "add_2" input: "add_672" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_198" op: "Sub" input: "bert/pooler/dense/kernel/read" input: "mul_1069" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_797" op: "Assign" input: "bert/pooler/dense/kernel" input: "sub_198" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_798" op: "Assign" input: "bert/pooler/dense/kernel/adam_m" input: "add_669" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_799" op: "Assign" input: "bert/pooler/dense/kernel/adam_v" input: "add_670" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/pooler/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/pooler/dense/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/pooler/dense/bias/adam_m/Assign" op: "Assign" input: "bert/pooler/dense/bias/adam_m" input: "bert/pooler/dense/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/pooler/dense/bias/adam_m/read" op: "Identity" input: "bert/pooler/dense/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "bert/pooler/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/pooler/dense/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "bert/pooler/dense/bias/adam_v/Assign" op: "Assign" input: "bert/pooler/dense/bias/adam_v" input: "bert/pooler/dense/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "bert/pooler/dense/bias/adam_v/read" op: "Identity" input: "bert/pooler/dense/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1070/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_1070" op: "Mul" input: "Mul_1070/x" input: "bert/pooler/dense/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1071/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_1071" op: "Mul" input: "Mul_1071/x" input: "clip_by_global_norm/clip_by_global_norm/_198" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_673" op: "Add" input: "Mul_1070" input: "Mul_1071" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1072/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_1072" op: "Mul" input: "Mul_1072/x" input: "bert/pooler/dense/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_198" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_198" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1073/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_1073" op: "Mul" input: "Mul_1073/x" input: "Square_198" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_674" op: "Add" input: "Mul_1072" input: "Mul_1073" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_198" op: "Sqrt" input: "add_674" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_675/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_675" op: "Add" input: "Sqrt_198" input: "add_675/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_199" op: "RealDiv" input: "add_673" input: "add_675" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_1074" op: "Mul" input: "add_2" input: "truediv_199" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_199" op: "Sub" input: "bert/pooler/dense/bias/read" input: "mul_1074" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_800" op: "Assign" input: "bert/pooler/dense/bias" input: "sub_199" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_801" op: "Assign" input: "bert/pooler/dense/bias/adam_m" input: "add_673" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_802" op: "Assign" input: "bert/pooler/dense/bias/adam_v" input: "add_674" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "cls/predictions/transform/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "cls/predictions/transform/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "cls/predictions/transform/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "cls/predictions/transform/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "cls/predictions/transform/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "cls/predictions/transform/dense/kernel/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "cls/predictions/transform/dense/kernel/adam_m/Assign" op: "Assign" input: "cls/predictions/transform/dense/kernel/adam_m" input: "cls/predictions/transform/dense/kernel/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "cls/predictions/transform/dense/kernel/adam_m/read" op: "Identity" input: "cls/predictions/transform/dense/kernel/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "cls/predictions/transform/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "cls/predictions/transform/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "cls/predictions/transform/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "cls/predictions/transform/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "cls/predictions/transform/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "cls/predictions/transform/dense/kernel/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "cls/predictions/transform/dense/kernel/adam_v/Assign" op: "Assign" input: "cls/predictions/transform/dense/kernel/adam_v" input: "cls/predictions/transform/dense/kernel/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "cls/predictions/transform/dense/kernel/adam_v/read" op: "Identity" input: "cls/predictions/transform/dense/kernel/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_1075/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_1075" op: "Mul" input: "Mul_1075/x" input: "cls/predictions/transform/dense/kernel/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_1076/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_1076" op: "Mul" input: "Mul_1076/x" input: "clip_by_global_norm/clip_by_global_norm/_199" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_676" op: "Add" input: "Mul_1075" input: "Mul_1076" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_1077/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_1077" op: "Mul" input: "Mul_1077/x" input: "cls/predictions/transform/dense/kernel/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_199" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_199" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_1078/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_1078" op: "Mul" input: "Mul_1078/x" input: "Square_199" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_677" op: "Add" input: "Mul_1077" input: "Mul_1078" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_199" op: "Sqrt" input: "add_677" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_678/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_678" op: "Add" input: "Sqrt_199" input: "add_678/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_200" op: "RealDiv" input: "add_676" input: "add_678" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_1079/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_1079" op: "Mul" input: "mul_1079/x" input: "cls/predictions/transform/dense/kernel/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_679" op: "Add" input: "truediv_200" input: "mul_1079" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_1080" op: "Mul" input: "add_2" input: "add_679" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "sub_200" op: "Sub" input: "cls/predictions/transform/dense/kernel/read" input: "mul_1080" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Assign_803" op: "Assign" input: "cls/predictions/transform/dense/kernel" input: "sub_200" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_804" op: "Assign" input: "cls/predictions/transform/dense/kernel/adam_m" input: "add_676" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_805" op: "Assign" input: "cls/predictions/transform/dense/kernel/adam_v" input: "add_677" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "cls/predictions/transform/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "cls/predictions/transform/dense/bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "cls/predictions/transform/dense/bias/adam_m/Assign" op: "Assign" input: "cls/predictions/transform/dense/bias/adam_m" input: "cls/predictions/transform/dense/bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "cls/predictions/transform/dense/bias/adam_m/read" op: "Identity" input: "cls/predictions/transform/dense/bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "cls/predictions/transform/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "cls/predictions/transform/dense/bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "cls/predictions/transform/dense/bias/adam_v/Assign" op: "Assign" input: "cls/predictions/transform/dense/bias/adam_v" input: "cls/predictions/transform/dense/bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "cls/predictions/transform/dense/bias/adam_v/read" op: "Identity" input: "cls/predictions/transform/dense/bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1081/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_1081" op: "Mul" input: "Mul_1081/x" input: "cls/predictions/transform/dense/bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1082/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_1082" op: "Mul" input: "Mul_1082/x" input: "clip_by_global_norm/clip_by_global_norm/_200" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_680" op: "Add" input: "Mul_1081" input: "Mul_1082" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1083/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_1083" op: "Mul" input: "Mul_1083/x" input: "cls/predictions/transform/dense/bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_200" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_200" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1084/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_1084" op: "Mul" input: "Mul_1084/x" input: "Square_200" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_681" op: "Add" input: "Mul_1083" input: "Mul_1084" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_200" op: "Sqrt" input: "add_681" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_682/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_682" op: "Add" input: "Sqrt_200" input: "add_682/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_201" op: "RealDiv" input: "add_680" input: "add_682" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_1085" op: "Mul" input: "add_2" input: "truediv_201" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_201" op: "Sub" input: "cls/predictions/transform/dense/bias/read" input: "mul_1085" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_806" op: "Assign" input: "cls/predictions/transform/dense/bias" input: "sub_201" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_807" op: "Assign" input: "cls/predictions/transform/dense/bias/adam_m" input: "add_680" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_808" op: "Assign" input: "cls/predictions/transform/dense/bias/adam_v" input: "add_681" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "cls/predictions/transform/LayerNorm/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "cls/predictions/transform/LayerNorm/beta/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "cls/predictions/transform/LayerNorm/beta/adam_m/Assign" op: "Assign" input: "cls/predictions/transform/LayerNorm/beta/adam_m" input: "cls/predictions/transform/LayerNorm/beta/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "cls/predictions/transform/LayerNorm/beta/adam_m/read" op: "Identity" input: "cls/predictions/transform/LayerNorm/beta/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "cls/predictions/transform/LayerNorm/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "cls/predictions/transform/LayerNorm/beta/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "cls/predictions/transform/LayerNorm/beta/adam_v/Assign" op: "Assign" input: "cls/predictions/transform/LayerNorm/beta/adam_v" input: "cls/predictions/transform/LayerNorm/beta/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "cls/predictions/transform/LayerNorm/beta/adam_v/read" op: "Identity" input: "cls/predictions/transform/LayerNorm/beta/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1086/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_1086" op: "Mul" input: "Mul_1086/x" input: "cls/predictions/transform/LayerNorm/beta/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1087/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_1087" op: "Mul" input: "Mul_1087/x" input: "clip_by_global_norm/clip_by_global_norm/_201" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_683" op: "Add" input: "Mul_1086" input: "Mul_1087" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1088/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_1088" op: "Mul" input: "Mul_1088/x" input: "cls/predictions/transform/LayerNorm/beta/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_201" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_201" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1089/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_1089" op: "Mul" input: "Mul_1089/x" input: "Square_201" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_684" op: "Add" input: "Mul_1088" input: "Mul_1089" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_201" op: "Sqrt" input: "add_684" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_685/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_685" op: "Add" input: "Sqrt_201" input: "add_685/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_202" op: "RealDiv" input: "add_683" input: "add_685" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_1090" op: "Mul" input: "add_2" input: "truediv_202" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_202" op: "Sub" input: "cls/predictions/transform/LayerNorm/beta/read" input: "mul_1090" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_809" op: "Assign" input: "cls/predictions/transform/LayerNorm/beta" input: "sub_202" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_810" op: "Assign" input: "cls/predictions/transform/LayerNorm/beta/adam_m" input: "add_683" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_811" op: "Assign" input: "cls/predictions/transform/LayerNorm/beta/adam_v" input: "add_684" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "cls/predictions/transform/LayerNorm/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "cls/predictions/transform/LayerNorm/gamma/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "cls/predictions/transform/LayerNorm/gamma/adam_m/Assign" op: "Assign" input: "cls/predictions/transform/LayerNorm/gamma/adam_m" input: "cls/predictions/transform/LayerNorm/gamma/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "cls/predictions/transform/LayerNorm/gamma/adam_m/read" op: "Identity" input: "cls/predictions/transform/LayerNorm/gamma/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "cls/predictions/transform/LayerNorm/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "cls/predictions/transform/LayerNorm/gamma/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "cls/predictions/transform/LayerNorm/gamma/adam_v/Assign" op: "Assign" input: "cls/predictions/transform/LayerNorm/gamma/adam_v" input: "cls/predictions/transform/LayerNorm/gamma/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "cls/predictions/transform/LayerNorm/gamma/adam_v/read" op: "Identity" input: "cls/predictions/transform/LayerNorm/gamma/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1091/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_1091" op: "Mul" input: "Mul_1091/x" input: "cls/predictions/transform/LayerNorm/gamma/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1092/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_1092" op: "Mul" input: "Mul_1092/x" input: "clip_by_global_norm/clip_by_global_norm/_202" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_686" op: "Add" input: "Mul_1091" input: "Mul_1092" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1093/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_1093" op: "Mul" input: "Mul_1093/x" input: "cls/predictions/transform/LayerNorm/gamma/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_202" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_202" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1094/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_1094" op: "Mul" input: "Mul_1094/x" input: "Square_202" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_687" op: "Add" input: "Mul_1093" input: "Mul_1094" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_202" op: "Sqrt" input: "add_687" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_688/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_688" op: "Add" input: "Sqrt_202" input: "add_688/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_203" op: "RealDiv" input: "add_686" input: "add_688" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_1095" op: "Mul" input: "add_2" input: "truediv_203" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "sub_203" op: "Sub" input: "cls/predictions/transform/LayerNorm/gamma/read" input: "mul_1095" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Assign_812" op: "Assign" input: "cls/predictions/transform/LayerNorm/gamma" input: "sub_203" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_813" op: "Assign" input: "cls/predictions/transform/LayerNorm/gamma/adam_m" input: "add_686" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_814" op: "Assign" input: "cls/predictions/transform/LayerNorm/gamma/adam_v" input: "add_687" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "cls/predictions/output_bias/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@cls/predictions/output_bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 28996 } } } } node { name: "cls/predictions/output_bias/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@cls/predictions/output_bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "cls/predictions/output_bias/adam_m/Initializer/zeros" op: "Fill" input: "cls/predictions/output_bias/adam_m/Initializer/zeros/shape_as_tensor" input: "cls/predictions/output_bias/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/output_bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "cls/predictions/output_bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@cls/predictions/output_bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 28996 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "cls/predictions/output_bias/adam_m/Assign" op: "Assign" input: "cls/predictions/output_bias/adam_m" input: "cls/predictions/output_bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/output_bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "cls/predictions/output_bias/adam_m/read" op: "Identity" input: "cls/predictions/output_bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/output_bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } } } } } } node { name: "cls/predictions/output_bias/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@cls/predictions/output_bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 28996 } } } } node { name: "cls/predictions/output_bias/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@cls/predictions/output_bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "cls/predictions/output_bias/adam_v/Initializer/zeros" op: "Fill" input: "cls/predictions/output_bias/adam_v/Initializer/zeros/shape_as_tensor" input: "cls/predictions/output_bias/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/output_bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "cls/predictions/output_bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@cls/predictions/output_bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 28996 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "cls/predictions/output_bias/adam_v/Assign" op: "Assign" input: "cls/predictions/output_bias/adam_v" input: "cls/predictions/output_bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/output_bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "cls/predictions/output_bias/adam_v/read" op: "Identity" input: "cls/predictions/output_bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/output_bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } } } } } } node { name: "Mul_1096/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_1096" op: "Mul" input: "Mul_1096/x" input: "cls/predictions/output_bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } } } } } } node { name: "Mul_1097/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_1097" op: "Mul" input: "Mul_1097/x" input: "clip_by_global_norm/clip_by_global_norm/_203" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } } } } } } node { name: "add_689" op: "Add" input: "Mul_1096" input: "Mul_1097" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } } } } } } node { name: "Mul_1098/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_1098" op: "Mul" input: "Mul_1098/x" input: "cls/predictions/output_bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } } } } } } node { name: "Square_203" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_203" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } } } } } } node { name: "Mul_1099/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_1099" op: "Mul" input: "Mul_1099/x" input: "Square_203" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } } } } } } node { name: "add_690" op: "Add" input: "Mul_1098" input: "Mul_1099" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } } } } } } node { name: "Sqrt_203" op: "Sqrt" input: "add_690" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } } } } } } node { name: "add_691/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_691" op: "Add" input: "Sqrt_203" input: "add_691/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } } } } } } node { name: "truediv_204" op: "RealDiv" input: "add_689" input: "add_691" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } } } } } } node { name: "mul_1100" op: "Mul" input: "add_2" input: "truediv_204" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } } } } } } node { name: "sub_204" op: "Sub" input: "cls/predictions/output_bias/read" input: "mul_1100" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } } } } } } node { name: "Assign_815" op: "Assign" input: "cls/predictions/output_bias" input: "sub_204" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/output_bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_816" op: "Assign" input: "cls/predictions/output_bias/adam_m" input: "add_689" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/output_bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_817" op: "Assign" input: "cls/predictions/output_bias/adam_v" input: "add_690" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/output_bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "cls/seq_relationship/output_weights/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_weights/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\002\000\000\000\000\003\000\000" } } } } node { name: "cls/seq_relationship/output_weights/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_weights/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "cls/seq_relationship/output_weights/adam_m/Initializer/zeros" op: "Fill" input: "cls/seq_relationship/output_weights/adam_m/Initializer/zeros/shape_as_tensor" input: "cls/seq_relationship/output_weights/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_weights/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "cls/seq_relationship/output_weights/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_weights/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 2 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "cls/seq_relationship/output_weights/adam_m/Assign" op: "Assign" input: "cls/seq_relationship/output_weights/adam_m" input: "cls/seq_relationship/output_weights/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_weights/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "cls/seq_relationship/output_weights/adam_m/read" op: "Identity" input: "cls/seq_relationship/output_weights/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_weights/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "cls/seq_relationship/output_weights/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_weights/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\002\000\000\000\000\003\000\000" } } } } node { name: "cls/seq_relationship/output_weights/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_weights/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "cls/seq_relationship/output_weights/adam_v/Initializer/zeros" op: "Fill" input: "cls/seq_relationship/output_weights/adam_v/Initializer/zeros/shape_as_tensor" input: "cls/seq_relationship/output_weights/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_weights/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "cls/seq_relationship/output_weights/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_weights/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 2 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "cls/seq_relationship/output_weights/adam_v/Assign" op: "Assign" input: "cls/seq_relationship/output_weights/adam_v" input: "cls/seq_relationship/output_weights/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_weights/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "cls/seq_relationship/output_weights/adam_v/read" op: "Identity" input: "cls/seq_relationship/output_weights/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_weights/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "Mul_1101/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_1101" op: "Mul" input: "Mul_1101/x" input: "cls/seq_relationship/output_weights/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "Mul_1102/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_1102" op: "Mul" input: "Mul_1102/x" input: "clip_by_global_norm/clip_by_global_norm/_204" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "add_692" op: "Add" input: "Mul_1101" input: "Mul_1102" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "Mul_1103/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_1103" op: "Mul" input: "Mul_1103/x" input: "cls/seq_relationship/output_weights/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "Square_204" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_204" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "Mul_1104/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_1104" op: "Mul" input: "Mul_1104/x" input: "Square_204" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "add_693" op: "Add" input: "Mul_1103" input: "Mul_1104" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "Sqrt_204" op: "Sqrt" input: "add_693" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "add_694/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_694" op: "Add" input: "Sqrt_204" input: "add_694/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "truediv_205" op: "RealDiv" input: "add_692" input: "add_694" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "mul_1105/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_1105" op: "Mul" input: "mul_1105/x" input: "cls/seq_relationship/output_weights/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "add_695" op: "Add" input: "truediv_205" input: "mul_1105" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "mul_1106" op: "Mul" input: "add_2" input: "add_695" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "sub_205" op: "Sub" input: "cls/seq_relationship/output_weights/read" input: "mul_1106" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "Assign_818" op: "Assign" input: "cls/seq_relationship/output_weights" input: "sub_205" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_weights" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_819" op: "Assign" input: "cls/seq_relationship/output_weights/adam_m" input: "add_692" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_weights/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_820" op: "Assign" input: "cls/seq_relationship/output_weights/adam_v" input: "add_693" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_weights/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "cls/seq_relationship/output_bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 2 } } float_val: 0.0 } } } } node { name: "cls/seq_relationship/output_bias/adam_m" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 2 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "cls/seq_relationship/output_bias/adam_m/Assign" op: "Assign" input: "cls/seq_relationship/output_bias/adam_m" input: "cls/seq_relationship/output_bias/adam_m/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "cls/seq_relationship/output_bias/adam_m/read" op: "Identity" input: "cls/seq_relationship/output_bias/adam_m" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "cls/seq_relationship/output_bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 2 } } float_val: 0.0 } } } } node { name: "cls/seq_relationship/output_bias/adam_v" op: "VariableV2" attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 2 } } } } attr { key: "shared_name" value { s: "" } } } node { name: "cls/seq_relationship/output_bias/adam_v/Assign" op: "Assign" input: "cls/seq_relationship/output_bias/adam_v" input: "cls/seq_relationship/output_bias/adam_v/Initializer/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "cls/seq_relationship/output_bias/adam_v/read" op: "Identity" input: "cls/seq_relationship/output_bias/adam_v" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "Mul_1107/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_1107" op: "Mul" input: "Mul_1107/x" input: "cls/seq_relationship/output_bias/adam_m/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "Mul_1108/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_1108" op: "Mul" input: "Mul_1108/x" input: "clip_by_global_norm/clip_by_global_norm/_205" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "add_696" op: "Add" input: "Mul_1107" input: "Mul_1108" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "Mul_1109/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_1109" op: "Mul" input: "Mul_1109/x" input: "cls/seq_relationship/output_bias/adam_v/read" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "Square_205" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_205" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "Mul_1110/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_1110" op: "Mul" input: "Mul_1110/x" input: "Square_205" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "add_697" op: "Add" input: "Mul_1109" input: "Mul_1110" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "Sqrt_205" op: "Sqrt" input: "add_697" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "add_698/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_698" op: "Add" input: "Sqrt_205" input: "add_698/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "truediv_206" op: "RealDiv" input: "add_696" input: "add_698" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "mul_1111" op: "Mul" input: "add_2" input: "truediv_206" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "sub_206" op: "Sub" input: "cls/seq_relationship/output_bias/read" input: "mul_1111" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "Assign_821" op: "Assign" input: "cls/seq_relationship/output_bias" input: "sub_206" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_822" op: "Assign" input: "cls/seq_relationship/output_bias/adam_m" input: "add_696" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "Assign_823" op: "Assign" input: "cls/seq_relationship/output_bias/adam_v" input: "add_697" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "use_locking" value { b: false } } attr { key: "validate_shape" value { b: true } } } node { name: "group_deps" op: "NoOp" input: "^Assign_206" input: "^Assign_207" input: "^Assign_208" input: "^Assign_209" input: "^Assign_210" input: "^Assign_211" input: "^Assign_212" input: "^Assign_213" input: "^Assign_214" input: "^Assign_215" input: "^Assign_216" input: "^Assign_217" input: "^Assign_218" input: "^Assign_219" input: "^Assign_220" input: "^Assign_221" input: "^Assign_222" input: "^Assign_223" input: "^Assign_224" input: "^Assign_225" input: "^Assign_226" input: "^Assign_227" input: "^Assign_228" input: "^Assign_229" input: "^Assign_230" input: "^Assign_231" input: "^Assign_232" input: "^Assign_233" input: "^Assign_234" input: "^Assign_235" input: "^Assign_236" input: "^Assign_237" input: "^Assign_238" input: "^Assign_239" input: "^Assign_240" input: "^Assign_241" input: "^Assign_242" input: "^Assign_243" input: "^Assign_244" input: "^Assign_245" input: "^Assign_246" input: "^Assign_247" input: "^Assign_248" input: "^Assign_249" input: "^Assign_250" input: "^Assign_251" input: "^Assign_252" input: "^Assign_253" input: "^Assign_254" input: "^Assign_255" input: "^Assign_256" input: "^Assign_257" input: "^Assign_258" input: "^Assign_259" input: "^Assign_260" input: "^Assign_261" input: "^Assign_262" input: "^Assign_263" input: "^Assign_264" input: "^Assign_265" input: "^Assign_266" input: "^Assign_267" input: "^Assign_268" input: "^Assign_269" input: "^Assign_270" input: "^Assign_271" input: "^Assign_272" input: "^Assign_273" input: "^Assign_274" input: "^Assign_275" input: "^Assign_276" input: "^Assign_277" input: "^Assign_278" input: "^Assign_279" input: "^Assign_280" input: "^Assign_281" input: "^Assign_282" input: "^Assign_283" input: "^Assign_284" input: "^Assign_285" input: "^Assign_286" input: "^Assign_287" input: "^Assign_288" input: "^Assign_289" input: "^Assign_290" input: "^Assign_291" input: "^Assign_292" input: "^Assign_293" input: "^Assign_294" input: "^Assign_295" input: "^Assign_296" input: "^Assign_297" input: "^Assign_298" input: "^Assign_299" input: "^Assign_300" input: "^Assign_301" input: "^Assign_302" input: "^Assign_303" input: "^Assign_304" input: "^Assign_305" input: "^Assign_306" input: "^Assign_307" input: "^Assign_308" input: "^Assign_309" input: "^Assign_310" input: "^Assign_311" input: "^Assign_312" input: "^Assign_313" input: "^Assign_314" input: "^Assign_315" input: "^Assign_316" input: "^Assign_317" input: "^Assign_318" input: "^Assign_319" input: "^Assign_320" input: "^Assign_321" input: "^Assign_322" input: "^Assign_323" input: "^Assign_324" input: "^Assign_325" input: "^Assign_326" input: "^Assign_327" input: "^Assign_328" input: "^Assign_329" input: "^Assign_330" input: "^Assign_331" input: "^Assign_332" input: "^Assign_333" input: "^Assign_334" input: "^Assign_335" input: "^Assign_336" input: "^Assign_337" input: "^Assign_338" input: "^Assign_339" input: "^Assign_340" input: "^Assign_341" input: "^Assign_342" input: "^Assign_343" input: "^Assign_344" input: "^Assign_345" input: "^Assign_346" input: "^Assign_347" input: "^Assign_348" input: "^Assign_349" input: "^Assign_350" input: "^Assign_351" input: "^Assign_352" input: "^Assign_353" input: "^Assign_354" input: "^Assign_355" input: "^Assign_356" input: "^Assign_357" input: "^Assign_358" input: "^Assign_359" input: "^Assign_360" input: "^Assign_361" input: "^Assign_362" input: "^Assign_363" input: "^Assign_364" input: "^Assign_365" input: "^Assign_366" input: "^Assign_367" input: "^Assign_368" input: "^Assign_369" input: "^Assign_370" input: "^Assign_371" input: "^Assign_372" input: "^Assign_373" input: "^Assign_374" input: "^Assign_375" input: "^Assign_376" input: "^Assign_377" input: "^Assign_378" input: "^Assign_379" input: "^Assign_380" input: "^Assign_381" input: "^Assign_382" input: "^Assign_383" input: "^Assign_384" input: "^Assign_385" input: "^Assign_386" input: "^Assign_387" input: "^Assign_388" input: "^Assign_389" input: "^Assign_390" input: "^Assign_391" input: "^Assign_392" input: "^Assign_393" input: "^Assign_394" input: "^Assign_395" input: "^Assign_396" input: "^Assign_397" input: "^Assign_398" input: "^Assign_399" input: "^Assign_400" input: "^Assign_401" input: "^Assign_402" input: "^Assign_403" input: "^Assign_404" input: "^Assign_405" input: "^Assign_406" input: "^Assign_407" input: "^Assign_408" input: "^Assign_409" input: "^Assign_410" input: "^Assign_411" input: "^Assign_412" input: "^Assign_413" input: "^Assign_414" input: "^Assign_415" input: "^Assign_416" input: "^Assign_417" input: "^Assign_418" input: "^Assign_419" input: "^Assign_420" input: "^Assign_421" input: "^Assign_422" input: "^Assign_423" input: "^Assign_424" input: "^Assign_425" input: "^Assign_426" input: "^Assign_427" input: "^Assign_428" input: "^Assign_429" input: "^Assign_430" input: "^Assign_431" input: "^Assign_432" input: "^Assign_433" input: "^Assign_434" input: "^Assign_435" input: "^Assign_436" input: "^Assign_437" input: "^Assign_438" input: "^Assign_439" input: "^Assign_440" input: "^Assign_441" input: "^Assign_442" input: "^Assign_443" input: "^Assign_444" input: "^Assign_445" input: "^Assign_446" input: "^Assign_447" input: "^Assign_448" input: "^Assign_449" input: "^Assign_450" input: "^Assign_451" input: "^Assign_452" input: "^Assign_453" input: "^Assign_454" input: "^Assign_455" input: "^Assign_456" input: "^Assign_457" input: "^Assign_458" input: "^Assign_459" input: "^Assign_460" input: "^Assign_461" input: "^Assign_462" input: "^Assign_463" input: "^Assign_464" input: "^Assign_465" input: "^Assign_466" input: "^Assign_467" input: "^Assign_468" input: "^Assign_469" input: "^Assign_470" input: "^Assign_471" input: "^Assign_472" input: "^Assign_473" input: "^Assign_474" input: "^Assign_475" input: "^Assign_476" input: "^Assign_477" input: "^Assign_478" input: "^Assign_479" input: "^Assign_480" input: "^Assign_481" input: "^Assign_482" input: "^Assign_483" input: "^Assign_484" input: "^Assign_485" input: "^Assign_486" input: "^Assign_487" input: "^Assign_488" input: "^Assign_489" input: "^Assign_490" input: "^Assign_491" input: "^Assign_492" input: "^Assign_493" input: "^Assign_494" input: "^Assign_495" input: "^Assign_496" input: "^Assign_497" input: "^Assign_498" input: "^Assign_499" input: "^Assign_500" input: "^Assign_501" input: "^Assign_502" input: "^Assign_503" input: "^Assign_504" input: "^Assign_505" input: "^Assign_506" input: "^Assign_507" input: "^Assign_508" input: "^Assign_509" input: "^Assign_510" input: "^Assign_511" input: "^Assign_512" input: "^Assign_513" input: "^Assign_514" input: "^Assign_515" input: "^Assign_516" input: "^Assign_517" input: "^Assign_518" input: "^Assign_519" input: "^Assign_520" input: "^Assign_521" input: "^Assign_522" input: "^Assign_523" input: "^Assign_524" input: "^Assign_525" input: "^Assign_526" input: "^Assign_527" input: "^Assign_528" input: "^Assign_529" input: "^Assign_530" input: "^Assign_531" input: "^Assign_532" input: "^Assign_533" input: "^Assign_534" input: "^Assign_535" input: "^Assign_536" input: "^Assign_537" input: "^Assign_538" input: "^Assign_539" input: "^Assign_540" input: "^Assign_541" input: "^Assign_542" input: "^Assign_543" input: "^Assign_544" input: "^Assign_545" input: "^Assign_546" input: "^Assign_547" input: "^Assign_548" input: "^Assign_549" input: "^Assign_550" input: "^Assign_551" input: "^Assign_552" input: "^Assign_553" input: "^Assign_554" input: "^Assign_555" input: "^Assign_556" input: "^Assign_557" input: "^Assign_558" input: "^Assign_559" input: "^Assign_560" input: "^Assign_561" input: "^Assign_562" input: "^Assign_563" input: "^Assign_564" input: "^Assign_565" input: "^Assign_566" input: "^Assign_567" input: "^Assign_568" input: "^Assign_569" input: "^Assign_570" input: "^Assign_571" input: "^Assign_572" input: "^Assign_573" input: "^Assign_574" input: "^Assign_575" input: "^Assign_576" input: "^Assign_577" input: "^Assign_578" input: "^Assign_579" input: "^Assign_580" input: "^Assign_581" input: "^Assign_582" input: "^Assign_583" input: "^Assign_584" input: "^Assign_585" input: "^Assign_586" input: "^Assign_587" input: "^Assign_588" input: "^Assign_589" input: "^Assign_590" input: "^Assign_591" input: "^Assign_592" input: "^Assign_593" input: "^Assign_594" input: "^Assign_595" input: "^Assign_596" input: "^Assign_597" input: "^Assign_598" input: "^Assign_599" input: "^Assign_600" input: "^Assign_601" input: "^Assign_602" input: "^Assign_603" input: "^Assign_604" input: "^Assign_605" input: "^Assign_606" input: "^Assign_607" input: "^Assign_608" input: "^Assign_609" input: "^Assign_610" input: "^Assign_611" input: "^Assign_612" input: "^Assign_613" input: "^Assign_614" input: "^Assign_615" input: "^Assign_616" input: "^Assign_617" input: "^Assign_618" input: "^Assign_619" input: "^Assign_620" input: "^Assign_621" input: "^Assign_622" input: "^Assign_623" input: "^Assign_624" input: "^Assign_625" input: "^Assign_626" input: "^Assign_627" input: "^Assign_628" input: "^Assign_629" input: "^Assign_630" input: "^Assign_631" input: "^Assign_632" input: "^Assign_633" input: "^Assign_634" input: "^Assign_635" input: "^Assign_636" input: "^Assign_637" input: "^Assign_638" input: "^Assign_639" input: "^Assign_640" input: "^Assign_641" input: "^Assign_642" input: "^Assign_643" input: "^Assign_644" input: "^Assign_645" input: "^Assign_646" input: "^Assign_647" input: "^Assign_648" input: "^Assign_649" input: "^Assign_650" input: "^Assign_651" input: "^Assign_652" input: "^Assign_653" input: "^Assign_654" input: "^Assign_655" input: "^Assign_656" input: "^Assign_657" input: "^Assign_658" input: "^Assign_659" input: "^Assign_660" input: "^Assign_661" input: "^Assign_662" input: "^Assign_663" input: "^Assign_664" input: "^Assign_665" input: "^Assign_666" input: "^Assign_667" input: "^Assign_668" input: "^Assign_669" input: "^Assign_670" input: "^Assign_671" input: "^Assign_672" input: "^Assign_673" input: "^Assign_674" input: "^Assign_675" input: "^Assign_676" input: "^Assign_677" input: "^Assign_678" input: "^Assign_679" input: "^Assign_680" input: "^Assign_681" input: "^Assign_682" input: "^Assign_683" input: "^Assign_684" input: "^Assign_685" input: "^Assign_686" input: "^Assign_687" input: "^Assign_688" input: "^Assign_689" input: "^Assign_690" input: "^Assign_691" input: "^Assign_692" input: "^Assign_693" input: "^Assign_694" input: "^Assign_695" input: "^Assign_696" input: "^Assign_697" input: "^Assign_698" input: "^Assign_699" input: "^Assign_700" input: "^Assign_701" input: "^Assign_702" input: "^Assign_703" input: "^Assign_704" input: "^Assign_705" input: "^Assign_706" input: "^Assign_707" input: "^Assign_708" input: "^Assign_709" input: "^Assign_710" input: "^Assign_711" input: "^Assign_712" input: "^Assign_713" input: "^Assign_714" input: "^Assign_715" input: "^Assign_716" input: "^Assign_717" input: "^Assign_718" input: "^Assign_719" input: "^Assign_720" input: "^Assign_721" input: "^Assign_722" input: "^Assign_723" input: "^Assign_724" input: "^Assign_725" input: "^Assign_726" input: "^Assign_727" input: "^Assign_728" input: "^Assign_729" input: "^Assign_730" input: "^Assign_731" input: "^Assign_732" input: "^Assign_733" input: "^Assign_734" input: "^Assign_735" input: "^Assign_736" input: "^Assign_737" input: "^Assign_738" input: "^Assign_739" input: "^Assign_740" input: "^Assign_741" input: "^Assign_742" input: "^Assign_743" input: "^Assign_744" input: "^Assign_745" input: "^Assign_746" input: "^Assign_747" input: "^Assign_748" input: "^Assign_749" input: "^Assign_750" input: "^Assign_751" input: "^Assign_752" input: "^Assign_753" input: "^Assign_754" input: "^Assign_755" input: "^Assign_756" input: "^Assign_757" input: "^Assign_758" input: "^Assign_759" input: "^Assign_760" input: "^Assign_761" input: "^Assign_762" input: "^Assign_763" input: "^Assign_764" input: "^Assign_765" input: "^Assign_766" input: "^Assign_767" input: "^Assign_768" input: "^Assign_769" input: "^Assign_770" input: "^Assign_771" input: "^Assign_772" input: "^Assign_773" input: "^Assign_774" input: "^Assign_775" input: "^Assign_776" input: "^Assign_777" input: "^Assign_778" input: "^Assign_779" input: "^Assign_780" input: "^Assign_781" input: "^Assign_782" input: "^Assign_783" input: "^Assign_784" input: "^Assign_785" input: "^Assign_786" input: "^Assign_787" input: "^Assign_788" input: "^Assign_789" input: "^Assign_790" input: "^Assign_791" input: "^Assign_792" input: "^Assign_793" input: "^Assign_794" input: "^Assign_795" input: "^Assign_796" input: "^Assign_797" input: "^Assign_798" input: "^Assign_799" input: "^Assign_800" input: "^Assign_801" input: "^Assign_802" input: "^Assign_803" input: "^Assign_804" input: "^Assign_805" input: "^Assign_806" input: "^Assign_807" input: "^Assign_808" input: "^Assign_809" input: "^Assign_810" input: "^Assign_811" input: "^Assign_812" input: "^Assign_813" input: "^Assign_814" input: "^Assign_815" input: "^Assign_816" input: "^Assign_817" input: "^Assign_818" input: "^Assign_819" input: "^Assign_820" input: "^Assign_821" input: "^Assign_822" input: "^Assign_823" } node { name: "ReadVariableOp" op: "ReadVariableOp" input: "global_step" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT64 } } } node { name: "add_699/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT64 } } attr { key: "value" value { tensor { dtype: DT_INT64 tensor_shape { } int64_val: 1 } } } } node { name: "add_699" op: "Add" input: "ReadVariableOp" input: "add_699/y" attr { key: "T" value { type: DT_INT64 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "AssignVariableOp" op: "AssignVariableOp" input: "global_step" input: "add_699" attr { key: "dtype" value { type: DT_INT64 } } } node { name: "ReadVariableOp_1" op: "ReadVariableOp" input: "global_step" input: "^AssignVariableOp" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT64 } } } node { name: "group_deps_1" op: "NoOp" input: "^ReadVariableOp_1" input: "^group_deps" } node { name: "loss/tags" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "loss" } } } } node { name: "loss" op: "ScalarSummary" input: "loss/tags" input: "add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "init" op: "NoOp" input: "^Assign" input: "^Assign_1" input: "^Assign_10" input: "^Assign_100" input: "^Assign_101" input: "^Assign_102" input: "^Assign_103" input: "^Assign_104" input: "^Assign_105" input: "^Assign_106" input: "^Assign_107" input: "^Assign_108" input: "^Assign_109" input: "^Assign_11" input: "^Assign_110" input: "^Assign_111" input: "^Assign_112" input: "^Assign_113" input: "^Assign_114" input: "^Assign_115" input: "^Assign_116" input: "^Assign_117" input: "^Assign_118" input: "^Assign_119" input: "^Assign_12" input: "^Assign_120" input: "^Assign_121" input: "^Assign_122" input: "^Assign_123" input: "^Assign_124" input: "^Assign_125" input: "^Assign_126" input: "^Assign_127" input: "^Assign_128" input: "^Assign_129" input: "^Assign_13" input: "^Assign_130" input: "^Assign_131" input: "^Assign_132" input: "^Assign_133" input: "^Assign_134" input: "^Assign_135" input: "^Assign_136" input: "^Assign_137" input: "^Assign_138" input: "^Assign_139" input: "^Assign_14" input: "^Assign_140" input: "^Assign_141" input: "^Assign_142" input: "^Assign_143" input: "^Assign_144" input: "^Assign_145" input: "^Assign_146" input: "^Assign_147" input: "^Assign_148" input: "^Assign_149" input: "^Assign_15" input: "^Assign_150" input: "^Assign_151" input: "^Assign_152" input: "^Assign_153" input: "^Assign_154" input: "^Assign_155" input: "^Assign_156" input: "^Assign_157" input: "^Assign_158" input: "^Assign_159" input: "^Assign_16" input: "^Assign_160" input: "^Assign_161" input: "^Assign_162" input: "^Assign_163" input: "^Assign_164" input: "^Assign_165" input: "^Assign_166" input: "^Assign_167" input: "^Assign_168" input: "^Assign_169" input: "^Assign_17" input: "^Assign_170" input: "^Assign_171" input: "^Assign_172" input: "^Assign_173" input: "^Assign_174" input: "^Assign_175" input: "^Assign_176" input: "^Assign_177" input: "^Assign_178" input: "^Assign_179" input: "^Assign_18" input: "^Assign_180" input: "^Assign_181" input: "^Assign_182" input: "^Assign_183" input: "^Assign_184" input: "^Assign_185" input: "^Assign_186" input: "^Assign_187" input: "^Assign_188" input: "^Assign_189" input: "^Assign_19" input: "^Assign_190" input: "^Assign_191" input: "^Assign_192" input: "^Assign_193" input: "^Assign_194" input: "^Assign_195" input: "^Assign_196" input: "^Assign_197" input: "^Assign_198" input: "^Assign_199" input: "^Assign_2" input: "^Assign_20" input: "^Assign_200" input: "^Assign_201" input: "^Assign_202" input: "^Assign_203" input: "^Assign_204" input: "^Assign_205" input: "^Assign_21" input: "^Assign_22" input: "^Assign_23" input: "^Assign_24" input: "^Assign_25" input: "^Assign_26" input: "^Assign_27" input: "^Assign_28" input: "^Assign_29" input: "^Assign_3" input: "^Assign_30" input: "^Assign_31" input: "^Assign_32" input: "^Assign_33" input: "^Assign_34" input: "^Assign_35" input: "^Assign_36" input: "^Assign_37" input: "^Assign_38" input: "^Assign_39" input: "^Assign_4" input: "^Assign_40" input: "^Assign_41" input: "^Assign_42" input: "^Assign_43" input: "^Assign_44" input: "^Assign_45" input: "^Assign_46" input: "^Assign_47" input: "^Assign_48" input: "^Assign_49" input: "^Assign_5" input: "^Assign_50" input: "^Assign_51" input: "^Assign_52" input: "^Assign_53" input: "^Assign_54" input: "^Assign_55" input: "^Assign_56" input: "^Assign_57" input: "^Assign_58" input: "^Assign_59" input: "^Assign_6" input: "^Assign_60" input: "^Assign_61" input: "^Assign_62" input: "^Assign_63" input: "^Assign_64" input: "^Assign_65" input: "^Assign_66" input: "^Assign_67" input: "^Assign_68" input: "^Assign_69" input: "^Assign_7" input: "^Assign_70" input: "^Assign_71" input: "^Assign_72" input: "^Assign_73" input: "^Assign_74" input: "^Assign_75" input: "^Assign_76" input: "^Assign_77" input: "^Assign_78" input: "^Assign_79" input: "^Assign_8" input: "^Assign_80" input: "^Assign_81" input: "^Assign_82" input: "^Assign_83" input: "^Assign_84" input: "^Assign_85" input: "^Assign_86" input: "^Assign_87" input: "^Assign_88" input: "^Assign_89" input: "^Assign_9" input: "^Assign_90" input: "^Assign_91" input: "^Assign_92" input: "^Assign_93" input: "^Assign_94" input: "^Assign_95" input: "^Assign_96" input: "^Assign_97" input: "^Assign_98" input: "^Assign_99" input: "^bert/embeddings/LayerNorm/beta/adam_m/Assign" input: "^bert/embeddings/LayerNorm/beta/adam_v/Assign" input: "^bert/embeddings/LayerNorm/gamma/adam_m/Assign" input: "^bert/embeddings/LayerNorm/gamma/adam_v/Assign" input: "^bert/embeddings/position_embeddings/adam_m/Assign" input: "^bert/embeddings/position_embeddings/adam_v/Assign" input: "^bert/embeddings/token_type_embeddings/adam_m/Assign" input: "^bert/embeddings/token_type_embeddings/adam_v/Assign" input: "^bert/embeddings/word_embeddings/adam_m/Assign" input: "^bert/embeddings/word_embeddings/adam_v/Assign" input: "^bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_m/Assign" input: "^bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_v/Assign" input: "^bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m/Assign" input: "^bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v/Assign" input: "^bert/encoder/layer_0/attention/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_0/attention/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_0/attention/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_0/attention/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_0/attention/self/key/bias/adam_m/Assign" input: "^bert/encoder/layer_0/attention/self/key/bias/adam_v/Assign" input: "^bert/encoder/layer_0/attention/self/key/kernel/adam_m/Assign" input: "^bert/encoder/layer_0/attention/self/key/kernel/adam_v/Assign" input: "^bert/encoder/layer_0/attention/self/query/bias/adam_m/Assign" input: "^bert/encoder/layer_0/attention/self/query/bias/adam_v/Assign" input: "^bert/encoder/layer_0/attention/self/query/kernel/adam_m/Assign" input: "^bert/encoder/layer_0/attention/self/query/kernel/adam_v/Assign" input: "^bert/encoder/layer_0/attention/self/value/bias/adam_m/Assign" input: "^bert/encoder/layer_0/attention/self/value/bias/adam_v/Assign" input: "^bert/encoder/layer_0/attention/self/value/kernel/adam_m/Assign" input: "^bert/encoder/layer_0/attention/self/value/kernel/adam_v/Assign" input: "^bert/encoder/layer_0/intermediate/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_0/intermediate/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_0/intermediate/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_0/intermediate/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_0/output/LayerNorm/beta/adam_m/Assign" input: "^bert/encoder/layer_0/output/LayerNorm/beta/adam_v/Assign" input: "^bert/encoder/layer_0/output/LayerNorm/gamma/adam_m/Assign" input: "^bert/encoder/layer_0/output/LayerNorm/gamma/adam_v/Assign" input: "^bert/encoder/layer_0/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_0/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_0/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_0/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_m/Assign" input: "^bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_v/Assign" input: "^bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m/Assign" input: "^bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v/Assign" input: "^bert/encoder/layer_1/attention/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_1/attention/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_1/attention/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_1/attention/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_1/attention/self/key/bias/adam_m/Assign" input: "^bert/encoder/layer_1/attention/self/key/bias/adam_v/Assign" input: "^bert/encoder/layer_1/attention/self/key/kernel/adam_m/Assign" input: "^bert/encoder/layer_1/attention/self/key/kernel/adam_v/Assign" input: "^bert/encoder/layer_1/attention/self/query/bias/adam_m/Assign" input: "^bert/encoder/layer_1/attention/self/query/bias/adam_v/Assign" input: "^bert/encoder/layer_1/attention/self/query/kernel/adam_m/Assign" input: "^bert/encoder/layer_1/attention/self/query/kernel/adam_v/Assign" input: "^bert/encoder/layer_1/attention/self/value/bias/adam_m/Assign" input: "^bert/encoder/layer_1/attention/self/value/bias/adam_v/Assign" input: "^bert/encoder/layer_1/attention/self/value/kernel/adam_m/Assign" input: "^bert/encoder/layer_1/attention/self/value/kernel/adam_v/Assign" input: "^bert/encoder/layer_1/intermediate/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_1/intermediate/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_1/intermediate/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_1/intermediate/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_1/output/LayerNorm/beta/adam_m/Assign" input: "^bert/encoder/layer_1/output/LayerNorm/beta/adam_v/Assign" input: "^bert/encoder/layer_1/output/LayerNorm/gamma/adam_m/Assign" input: "^bert/encoder/layer_1/output/LayerNorm/gamma/adam_v/Assign" input: "^bert/encoder/layer_1/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_1/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_1/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_1/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_m/Assign" input: "^bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_v/Assign" input: "^bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m/Assign" input: "^bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v/Assign" input: "^bert/encoder/layer_10/attention/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_10/attention/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_10/attention/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_10/attention/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_10/attention/self/key/bias/adam_m/Assign" input: "^bert/encoder/layer_10/attention/self/key/bias/adam_v/Assign" input: "^bert/encoder/layer_10/attention/self/key/kernel/adam_m/Assign" input: "^bert/encoder/layer_10/attention/self/key/kernel/adam_v/Assign" input: "^bert/encoder/layer_10/attention/self/query/bias/adam_m/Assign" input: "^bert/encoder/layer_10/attention/self/query/bias/adam_v/Assign" input: "^bert/encoder/layer_10/attention/self/query/kernel/adam_m/Assign" input: "^bert/encoder/layer_10/attention/self/query/kernel/adam_v/Assign" input: "^bert/encoder/layer_10/attention/self/value/bias/adam_m/Assign" input: "^bert/encoder/layer_10/attention/self/value/bias/adam_v/Assign" input: "^bert/encoder/layer_10/attention/self/value/kernel/adam_m/Assign" input: "^bert/encoder/layer_10/attention/self/value/kernel/adam_v/Assign" input: "^bert/encoder/layer_10/intermediate/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_10/intermediate/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_10/intermediate/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_10/intermediate/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_10/output/LayerNorm/beta/adam_m/Assign" input: "^bert/encoder/layer_10/output/LayerNorm/beta/adam_v/Assign" input: "^bert/encoder/layer_10/output/LayerNorm/gamma/adam_m/Assign" input: "^bert/encoder/layer_10/output/LayerNorm/gamma/adam_v/Assign" input: "^bert/encoder/layer_10/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_10/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_10/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_10/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_m/Assign" input: "^bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_v/Assign" input: "^bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m/Assign" input: "^bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v/Assign" input: "^bert/encoder/layer_11/attention/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_11/attention/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_11/attention/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_11/attention/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_11/attention/self/key/bias/adam_m/Assign" input: "^bert/encoder/layer_11/attention/self/key/bias/adam_v/Assign" input: "^bert/encoder/layer_11/attention/self/key/kernel/adam_m/Assign" input: "^bert/encoder/layer_11/attention/self/key/kernel/adam_v/Assign" input: "^bert/encoder/layer_11/attention/self/query/bias/adam_m/Assign" input: "^bert/encoder/layer_11/attention/self/query/bias/adam_v/Assign" input: "^bert/encoder/layer_11/attention/self/query/kernel/adam_m/Assign" input: "^bert/encoder/layer_11/attention/self/query/kernel/adam_v/Assign" input: "^bert/encoder/layer_11/attention/self/value/bias/adam_m/Assign" input: "^bert/encoder/layer_11/attention/self/value/bias/adam_v/Assign" input: "^bert/encoder/layer_11/attention/self/value/kernel/adam_m/Assign" input: "^bert/encoder/layer_11/attention/self/value/kernel/adam_v/Assign" input: "^bert/encoder/layer_11/intermediate/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_11/intermediate/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_11/intermediate/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_11/intermediate/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_11/output/LayerNorm/beta/adam_m/Assign" input: "^bert/encoder/layer_11/output/LayerNorm/beta/adam_v/Assign" input: "^bert/encoder/layer_11/output/LayerNorm/gamma/adam_m/Assign" input: "^bert/encoder/layer_11/output/LayerNorm/gamma/adam_v/Assign" input: "^bert/encoder/layer_11/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_11/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_11/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_11/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_m/Assign" input: "^bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_v/Assign" input: "^bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m/Assign" input: "^bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v/Assign" input: "^bert/encoder/layer_2/attention/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_2/attention/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_2/attention/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_2/attention/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_2/attention/self/key/bias/adam_m/Assign" input: "^bert/encoder/layer_2/attention/self/key/bias/adam_v/Assign" input: "^bert/encoder/layer_2/attention/self/key/kernel/adam_m/Assign" input: "^bert/encoder/layer_2/attention/self/key/kernel/adam_v/Assign" input: "^bert/encoder/layer_2/attention/self/query/bias/adam_m/Assign" input: "^bert/encoder/layer_2/attention/self/query/bias/adam_v/Assign" input: "^bert/encoder/layer_2/attention/self/query/kernel/adam_m/Assign" input: "^bert/encoder/layer_2/attention/self/query/kernel/adam_v/Assign" input: "^bert/encoder/layer_2/attention/self/value/bias/adam_m/Assign" input: "^bert/encoder/layer_2/attention/self/value/bias/adam_v/Assign" input: "^bert/encoder/layer_2/attention/self/value/kernel/adam_m/Assign" input: "^bert/encoder/layer_2/attention/self/value/kernel/adam_v/Assign" input: "^bert/encoder/layer_2/intermediate/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_2/intermediate/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_2/intermediate/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_2/intermediate/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_2/output/LayerNorm/beta/adam_m/Assign" input: "^bert/encoder/layer_2/output/LayerNorm/beta/adam_v/Assign" input: "^bert/encoder/layer_2/output/LayerNorm/gamma/adam_m/Assign" input: "^bert/encoder/layer_2/output/LayerNorm/gamma/adam_v/Assign" input: "^bert/encoder/layer_2/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_2/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_2/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_2/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_m/Assign" input: "^bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_v/Assign" input: "^bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m/Assign" input: "^bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v/Assign" input: "^bert/encoder/layer_3/attention/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_3/attention/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_3/attention/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_3/attention/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_3/attention/self/key/bias/adam_m/Assign" input: "^bert/encoder/layer_3/attention/self/key/bias/adam_v/Assign" input: "^bert/encoder/layer_3/attention/self/key/kernel/adam_m/Assign" input: "^bert/encoder/layer_3/attention/self/key/kernel/adam_v/Assign" input: "^bert/encoder/layer_3/attention/self/query/bias/adam_m/Assign" input: "^bert/encoder/layer_3/attention/self/query/bias/adam_v/Assign" input: "^bert/encoder/layer_3/attention/self/query/kernel/adam_m/Assign" input: "^bert/encoder/layer_3/attention/self/query/kernel/adam_v/Assign" input: "^bert/encoder/layer_3/attention/self/value/bias/adam_m/Assign" input: "^bert/encoder/layer_3/attention/self/value/bias/adam_v/Assign" input: "^bert/encoder/layer_3/attention/self/value/kernel/adam_m/Assign" input: "^bert/encoder/layer_3/attention/self/value/kernel/adam_v/Assign" input: "^bert/encoder/layer_3/intermediate/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_3/intermediate/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_3/intermediate/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_3/intermediate/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_3/output/LayerNorm/beta/adam_m/Assign" input: "^bert/encoder/layer_3/output/LayerNorm/beta/adam_v/Assign" input: "^bert/encoder/layer_3/output/LayerNorm/gamma/adam_m/Assign" input: "^bert/encoder/layer_3/output/LayerNorm/gamma/adam_v/Assign" input: "^bert/encoder/layer_3/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_3/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_3/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_3/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_m/Assign" input: "^bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_v/Assign" input: "^bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m/Assign" input: "^bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v/Assign" input: "^bert/encoder/layer_4/attention/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_4/attention/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_4/attention/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_4/attention/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_4/attention/self/key/bias/adam_m/Assign" input: "^bert/encoder/layer_4/attention/self/key/bias/adam_v/Assign" input: "^bert/encoder/layer_4/attention/self/key/kernel/adam_m/Assign" input: "^bert/encoder/layer_4/attention/self/key/kernel/adam_v/Assign" input: "^bert/encoder/layer_4/attention/self/query/bias/adam_m/Assign" input: "^bert/encoder/layer_4/attention/self/query/bias/adam_v/Assign" input: "^bert/encoder/layer_4/attention/self/query/kernel/adam_m/Assign" input: "^bert/encoder/layer_4/attention/self/query/kernel/adam_v/Assign" input: "^bert/encoder/layer_4/attention/self/value/bias/adam_m/Assign" input: "^bert/encoder/layer_4/attention/self/value/bias/adam_v/Assign" input: "^bert/encoder/layer_4/attention/self/value/kernel/adam_m/Assign" input: "^bert/encoder/layer_4/attention/self/value/kernel/adam_v/Assign" input: "^bert/encoder/layer_4/intermediate/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_4/intermediate/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_4/intermediate/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_4/intermediate/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_4/output/LayerNorm/beta/adam_m/Assign" input: "^bert/encoder/layer_4/output/LayerNorm/beta/adam_v/Assign" input: "^bert/encoder/layer_4/output/LayerNorm/gamma/adam_m/Assign" input: "^bert/encoder/layer_4/output/LayerNorm/gamma/adam_v/Assign" input: "^bert/encoder/layer_4/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_4/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_4/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_4/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_m/Assign" input: "^bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_v/Assign" input: "^bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m/Assign" input: "^bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v/Assign" input: "^bert/encoder/layer_5/attention/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_5/attention/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_5/attention/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_5/attention/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_5/attention/self/key/bias/adam_m/Assign" input: "^bert/encoder/layer_5/attention/self/key/bias/adam_v/Assign" input: "^bert/encoder/layer_5/attention/self/key/kernel/adam_m/Assign" input: "^bert/encoder/layer_5/attention/self/key/kernel/adam_v/Assign" input: "^bert/encoder/layer_5/attention/self/query/bias/adam_m/Assign" input: "^bert/encoder/layer_5/attention/self/query/bias/adam_v/Assign" input: "^bert/encoder/layer_5/attention/self/query/kernel/adam_m/Assign" input: "^bert/encoder/layer_5/attention/self/query/kernel/adam_v/Assign" input: "^bert/encoder/layer_5/attention/self/value/bias/adam_m/Assign" input: "^bert/encoder/layer_5/attention/self/value/bias/adam_v/Assign" input: "^bert/encoder/layer_5/attention/self/value/kernel/adam_m/Assign" input: "^bert/encoder/layer_5/attention/self/value/kernel/adam_v/Assign" input: "^bert/encoder/layer_5/intermediate/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_5/intermediate/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_5/intermediate/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_5/intermediate/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_5/output/LayerNorm/beta/adam_m/Assign" input: "^bert/encoder/layer_5/output/LayerNorm/beta/adam_v/Assign" input: "^bert/encoder/layer_5/output/LayerNorm/gamma/adam_m/Assign" input: "^bert/encoder/layer_5/output/LayerNorm/gamma/adam_v/Assign" input: "^bert/encoder/layer_5/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_5/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_5/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_5/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_m/Assign" input: "^bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_v/Assign" input: "^bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m/Assign" input: "^bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v/Assign" input: "^bert/encoder/layer_6/attention/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_6/attention/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_6/attention/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_6/attention/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_6/attention/self/key/bias/adam_m/Assign" input: "^bert/encoder/layer_6/attention/self/key/bias/adam_v/Assign" input: "^bert/encoder/layer_6/attention/self/key/kernel/adam_m/Assign" input: "^bert/encoder/layer_6/attention/self/key/kernel/adam_v/Assign" input: "^bert/encoder/layer_6/attention/self/query/bias/adam_m/Assign" input: "^bert/encoder/layer_6/attention/self/query/bias/adam_v/Assign" input: "^bert/encoder/layer_6/attention/self/query/kernel/adam_m/Assign" input: "^bert/encoder/layer_6/attention/self/query/kernel/adam_v/Assign" input: "^bert/encoder/layer_6/attention/self/value/bias/adam_m/Assign" input: "^bert/encoder/layer_6/attention/self/value/bias/adam_v/Assign" input: "^bert/encoder/layer_6/attention/self/value/kernel/adam_m/Assign" input: "^bert/encoder/layer_6/attention/self/value/kernel/adam_v/Assign" input: "^bert/encoder/layer_6/intermediate/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_6/intermediate/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_6/intermediate/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_6/intermediate/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_6/output/LayerNorm/beta/adam_m/Assign" input: "^bert/encoder/layer_6/output/LayerNorm/beta/adam_v/Assign" input: "^bert/encoder/layer_6/output/LayerNorm/gamma/adam_m/Assign" input: "^bert/encoder/layer_6/output/LayerNorm/gamma/adam_v/Assign" input: "^bert/encoder/layer_6/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_6/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_6/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_6/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_m/Assign" input: "^bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_v/Assign" input: "^bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m/Assign" input: "^bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v/Assign" input: "^bert/encoder/layer_7/attention/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_7/attention/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_7/attention/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_7/attention/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_7/attention/self/key/bias/adam_m/Assign" input: "^bert/encoder/layer_7/attention/self/key/bias/adam_v/Assign" input: "^bert/encoder/layer_7/attention/self/key/kernel/adam_m/Assign" input: "^bert/encoder/layer_7/attention/self/key/kernel/adam_v/Assign" input: "^bert/encoder/layer_7/attention/self/query/bias/adam_m/Assign" input: "^bert/encoder/layer_7/attention/self/query/bias/adam_v/Assign" input: "^bert/encoder/layer_7/attention/self/query/kernel/adam_m/Assign" input: "^bert/encoder/layer_7/attention/self/query/kernel/adam_v/Assign" input: "^bert/encoder/layer_7/attention/self/value/bias/adam_m/Assign" input: "^bert/encoder/layer_7/attention/self/value/bias/adam_v/Assign" input: "^bert/encoder/layer_7/attention/self/value/kernel/adam_m/Assign" input: "^bert/encoder/layer_7/attention/self/value/kernel/adam_v/Assign" input: "^bert/encoder/layer_7/intermediate/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_7/intermediate/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_7/intermediate/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_7/intermediate/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_7/output/LayerNorm/beta/adam_m/Assign" input: "^bert/encoder/layer_7/output/LayerNorm/beta/adam_v/Assign" input: "^bert/encoder/layer_7/output/LayerNorm/gamma/adam_m/Assign" input: "^bert/encoder/layer_7/output/LayerNorm/gamma/adam_v/Assign" input: "^bert/encoder/layer_7/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_7/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_7/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_7/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_m/Assign" input: "^bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_v/Assign" input: "^bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m/Assign" input: "^bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v/Assign" input: "^bert/encoder/layer_8/attention/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_8/attention/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_8/attention/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_8/attention/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_8/attention/self/key/bias/adam_m/Assign" input: "^bert/encoder/layer_8/attention/self/key/bias/adam_v/Assign" input: "^bert/encoder/layer_8/attention/self/key/kernel/adam_m/Assign" input: "^bert/encoder/layer_8/attention/self/key/kernel/adam_v/Assign" input: "^bert/encoder/layer_8/attention/self/query/bias/adam_m/Assign" input: "^bert/encoder/layer_8/attention/self/query/bias/adam_v/Assign" input: "^bert/encoder/layer_8/attention/self/query/kernel/adam_m/Assign" input: "^bert/encoder/layer_8/attention/self/query/kernel/adam_v/Assign" input: "^bert/encoder/layer_8/attention/self/value/bias/adam_m/Assign" input: "^bert/encoder/layer_8/attention/self/value/bias/adam_v/Assign" input: "^bert/encoder/layer_8/attention/self/value/kernel/adam_m/Assign" input: "^bert/encoder/layer_8/attention/self/value/kernel/adam_v/Assign" input: "^bert/encoder/layer_8/intermediate/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_8/intermediate/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_8/intermediate/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_8/intermediate/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_8/output/LayerNorm/beta/adam_m/Assign" input: "^bert/encoder/layer_8/output/LayerNorm/beta/adam_v/Assign" input: "^bert/encoder/layer_8/output/LayerNorm/gamma/adam_m/Assign" input: "^bert/encoder/layer_8/output/LayerNorm/gamma/adam_v/Assign" input: "^bert/encoder/layer_8/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_8/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_8/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_8/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_m/Assign" input: "^bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_v/Assign" input: "^bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m/Assign" input: "^bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v/Assign" input: "^bert/encoder/layer_9/attention/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_9/attention/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_9/attention/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_9/attention/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_9/attention/self/key/bias/adam_m/Assign" input: "^bert/encoder/layer_9/attention/self/key/bias/adam_v/Assign" input: "^bert/encoder/layer_9/attention/self/key/kernel/adam_m/Assign" input: "^bert/encoder/layer_9/attention/self/key/kernel/adam_v/Assign" input: "^bert/encoder/layer_9/attention/self/query/bias/adam_m/Assign" input: "^bert/encoder/layer_9/attention/self/query/bias/adam_v/Assign" input: "^bert/encoder/layer_9/attention/self/query/kernel/adam_m/Assign" input: "^bert/encoder/layer_9/attention/self/query/kernel/adam_v/Assign" input: "^bert/encoder/layer_9/attention/self/value/bias/adam_m/Assign" input: "^bert/encoder/layer_9/attention/self/value/bias/adam_v/Assign" input: "^bert/encoder/layer_9/attention/self/value/kernel/adam_m/Assign" input: "^bert/encoder/layer_9/attention/self/value/kernel/adam_v/Assign" input: "^bert/encoder/layer_9/intermediate/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_9/intermediate/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_9/intermediate/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_9/intermediate/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_9/output/LayerNorm/beta/adam_m/Assign" input: "^bert/encoder/layer_9/output/LayerNorm/beta/adam_v/Assign" input: "^bert/encoder/layer_9/output/LayerNorm/gamma/adam_m/Assign" input: "^bert/encoder/layer_9/output/LayerNorm/gamma/adam_v/Assign" input: "^bert/encoder/layer_9/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_9/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_9/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_9/output/dense/kernel/adam_v/Assign" input: "^bert/pooler/dense/bias/adam_m/Assign" input: "^bert/pooler/dense/bias/adam_v/Assign" input: "^bert/pooler/dense/kernel/adam_m/Assign" input: "^bert/pooler/dense/kernel/adam_v/Assign" input: "^cls/predictions/output_bias/adam_m/Assign" input: "^cls/predictions/output_bias/adam_v/Assign" input: "^cls/predictions/transform/LayerNorm/beta/adam_m/Assign" input: "^cls/predictions/transform/LayerNorm/beta/adam_v/Assign" input: "^cls/predictions/transform/LayerNorm/gamma/adam_m/Assign" input: "^cls/predictions/transform/LayerNorm/gamma/adam_v/Assign" input: "^cls/predictions/transform/dense/bias/adam_m/Assign" input: "^cls/predictions/transform/dense/bias/adam_v/Assign" input: "^cls/predictions/transform/dense/kernel/adam_m/Assign" input: "^cls/predictions/transform/dense/kernel/adam_v/Assign" input: "^cls/seq_relationship/output_bias/adam_m/Assign" input: "^cls/seq_relationship/output_bias/adam_v/Assign" input: "^cls/seq_relationship/output_weights/adam_m/Assign" input: "^cls/seq_relationship/output_weights/adam_v/Assign" input: "^global_step/Assign" } node { name: "init_1" op: "NoOp" } node { name: "group_deps_2" op: "NoOp" input: "^init" input: "^init_1" } node { name: "report_uninitialized_variables/VarIsInitializedOp" op: "VarIsInitializedOp" input: "global_step" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/IsVariableInitialized" op: "IsVariableInitialized" input: "bert/embeddings/word_embeddings" attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_1" op: "IsVariableInitialized" input: "bert/embeddings/token_type_embeddings" attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_2" op: "IsVariableInitialized" input: "bert/embeddings/position_embeddings" attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_3" op: "IsVariableInitialized" input: "bert/embeddings/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_4" op: "IsVariableInitialized" input: "bert/embeddings/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_5" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/self/query/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_6" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/self/query/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_7" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/self/key/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_8" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/self/key/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_9" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/self/value/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_10" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/self/value/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_11" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_12" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_13" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_14" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_15" op: "IsVariableInitialized" input: "bert/encoder/layer_0/intermediate/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_16" op: "IsVariableInitialized" input: "bert/encoder/layer_0/intermediate/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_17" op: "IsVariableInitialized" input: "bert/encoder/layer_0/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_18" op: "IsVariableInitialized" input: "bert/encoder/layer_0/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_19" op: "IsVariableInitialized" input: "bert/encoder/layer_0/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_20" op: "IsVariableInitialized" input: "bert/encoder/layer_0/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_21" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/self/query/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_22" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/self/query/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_23" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/self/key/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_24" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/self/key/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_25" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/self/value/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_26" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/self/value/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_27" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_28" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_29" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_30" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_31" op: "IsVariableInitialized" input: "bert/encoder/layer_1/intermediate/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_32" op: "IsVariableInitialized" input: "bert/encoder/layer_1/intermediate/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_33" op: "IsVariableInitialized" input: "bert/encoder/layer_1/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_34" op: "IsVariableInitialized" input: "bert/encoder/layer_1/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_35" op: "IsVariableInitialized" input: "bert/encoder/layer_1/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_36" op: "IsVariableInitialized" input: "bert/encoder/layer_1/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_37" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/self/query/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_38" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/self/query/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_39" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/self/key/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_40" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/self/key/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_41" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/self/value/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_42" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/self/value/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_43" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_44" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_45" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_46" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_47" op: "IsVariableInitialized" input: "bert/encoder/layer_2/intermediate/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_48" op: "IsVariableInitialized" input: "bert/encoder/layer_2/intermediate/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_49" op: "IsVariableInitialized" input: "bert/encoder/layer_2/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_50" op: "IsVariableInitialized" input: "bert/encoder/layer_2/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_51" op: "IsVariableInitialized" input: "bert/encoder/layer_2/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_52" op: "IsVariableInitialized" input: "bert/encoder/layer_2/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_53" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/self/query/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_54" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/self/query/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_55" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/self/key/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_56" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/self/key/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_57" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/self/value/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_58" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/self/value/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_59" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_60" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_61" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_62" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_63" op: "IsVariableInitialized" input: "bert/encoder/layer_3/intermediate/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_64" op: "IsVariableInitialized" input: "bert/encoder/layer_3/intermediate/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_65" op: "IsVariableInitialized" input: "bert/encoder/layer_3/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_66" op: "IsVariableInitialized" input: "bert/encoder/layer_3/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_67" op: "IsVariableInitialized" input: "bert/encoder/layer_3/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_68" op: "IsVariableInitialized" input: "bert/encoder/layer_3/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_69" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/self/query/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_70" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/self/query/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_71" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/self/key/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_72" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/self/key/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_73" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/self/value/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_74" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/self/value/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_75" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_76" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_77" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_78" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_79" op: "IsVariableInitialized" input: "bert/encoder/layer_4/intermediate/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_80" op: "IsVariableInitialized" input: "bert/encoder/layer_4/intermediate/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_81" op: "IsVariableInitialized" input: "bert/encoder/layer_4/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_82" op: "IsVariableInitialized" input: "bert/encoder/layer_4/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_83" op: "IsVariableInitialized" input: "bert/encoder/layer_4/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_84" op: "IsVariableInitialized" input: "bert/encoder/layer_4/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_85" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/self/query/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_86" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/self/query/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_87" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/self/key/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_88" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/self/key/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_89" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/self/value/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_90" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/self/value/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_91" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_92" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_93" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_94" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_95" op: "IsVariableInitialized" input: "bert/encoder/layer_5/intermediate/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_96" op: "IsVariableInitialized" input: "bert/encoder/layer_5/intermediate/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_97" op: "IsVariableInitialized" input: "bert/encoder/layer_5/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_98" op: "IsVariableInitialized" input: "bert/encoder/layer_5/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_99" op: "IsVariableInitialized" input: "bert/encoder/layer_5/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_100" op: "IsVariableInitialized" input: "bert/encoder/layer_5/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_101" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/self/query/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_102" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/self/query/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_103" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/self/key/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_104" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/self/key/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_105" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/self/value/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_106" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/self/value/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_107" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_108" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_109" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_110" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_111" op: "IsVariableInitialized" input: "bert/encoder/layer_6/intermediate/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_112" op: "IsVariableInitialized" input: "bert/encoder/layer_6/intermediate/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_113" op: "IsVariableInitialized" input: "bert/encoder/layer_6/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_114" op: "IsVariableInitialized" input: "bert/encoder/layer_6/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_115" op: "IsVariableInitialized" input: "bert/encoder/layer_6/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_116" op: "IsVariableInitialized" input: "bert/encoder/layer_6/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_117" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/self/query/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_118" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/self/query/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_119" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/self/key/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_120" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/self/key/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_121" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/self/value/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_122" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/self/value/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_123" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_124" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_125" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_126" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_127" op: "IsVariableInitialized" input: "bert/encoder/layer_7/intermediate/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_128" op: "IsVariableInitialized" input: "bert/encoder/layer_7/intermediate/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_129" op: "IsVariableInitialized" input: "bert/encoder/layer_7/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_130" op: "IsVariableInitialized" input: "bert/encoder/layer_7/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_131" op: "IsVariableInitialized" input: "bert/encoder/layer_7/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_132" op: "IsVariableInitialized" input: "bert/encoder/layer_7/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_133" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/self/query/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_134" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/self/query/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_135" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/self/key/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_136" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/self/key/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_137" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/self/value/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_138" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/self/value/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_139" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_140" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_141" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_142" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_143" op: "IsVariableInitialized" input: "bert/encoder/layer_8/intermediate/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_144" op: "IsVariableInitialized" input: "bert/encoder/layer_8/intermediate/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_145" op: "IsVariableInitialized" input: "bert/encoder/layer_8/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_146" op: "IsVariableInitialized" input: "bert/encoder/layer_8/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_147" op: "IsVariableInitialized" input: "bert/encoder/layer_8/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_148" op: "IsVariableInitialized" input: "bert/encoder/layer_8/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_149" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/self/query/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_150" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/self/query/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_151" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/self/key/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_152" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/self/key/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_153" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/self/value/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_154" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/self/value/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_155" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_156" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_157" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_158" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_159" op: "IsVariableInitialized" input: "bert/encoder/layer_9/intermediate/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_160" op: "IsVariableInitialized" input: "bert/encoder/layer_9/intermediate/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_161" op: "IsVariableInitialized" input: "bert/encoder/layer_9/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_162" op: "IsVariableInitialized" input: "bert/encoder/layer_9/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_163" op: "IsVariableInitialized" input: "bert/encoder/layer_9/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_164" op: "IsVariableInitialized" input: "bert/encoder/layer_9/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_165" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/self/query/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_166" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/self/query/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_167" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/self/key/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_168" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/self/key/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_169" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/self/value/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_170" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/self/value/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_171" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_172" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_173" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_174" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_175" op: "IsVariableInitialized" input: "bert/encoder/layer_10/intermediate/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_176" op: "IsVariableInitialized" input: "bert/encoder/layer_10/intermediate/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_177" op: "IsVariableInitialized" input: "bert/encoder/layer_10/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_178" op: "IsVariableInitialized" input: "bert/encoder/layer_10/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_179" op: "IsVariableInitialized" input: "bert/encoder/layer_10/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_180" op: "IsVariableInitialized" input: "bert/encoder/layer_10/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_181" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/self/query/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_182" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/self/query/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_183" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/self/key/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_184" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/self/key/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_185" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/self/value/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_186" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/self/value/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_187" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_188" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_189" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_190" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_191" op: "IsVariableInitialized" input: "bert/encoder/layer_11/intermediate/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_192" op: "IsVariableInitialized" input: "bert/encoder/layer_11/intermediate/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_193" op: "IsVariableInitialized" input: "bert/encoder/layer_11/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_194" op: "IsVariableInitialized" input: "bert/encoder/layer_11/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_195" op: "IsVariableInitialized" input: "bert/encoder/layer_11/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_196" op: "IsVariableInitialized" input: "bert/encoder/layer_11/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_197" op: "IsVariableInitialized" input: "bert/pooler/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_198" op: "IsVariableInitialized" input: "bert/pooler/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_199" op: "IsVariableInitialized" input: "cls/predictions/transform/dense/kernel" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_200" op: "IsVariableInitialized" input: "cls/predictions/transform/dense/bias" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_201" op: "IsVariableInitialized" input: "cls/predictions/transform/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_202" op: "IsVariableInitialized" input: "cls/predictions/transform/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_203" op: "IsVariableInitialized" input: "cls/predictions/output_bias" attr { key: "_class" value { list { s: "loc:@cls/predictions/output_bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_204" op: "IsVariableInitialized" input: "cls/seq_relationship/output_weights" attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_weights" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_205" op: "IsVariableInitialized" input: "cls/seq_relationship/output_bias" attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_206" op: "IsVariableInitialized" input: "bert/embeddings/word_embeddings/adam_m" attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_207" op: "IsVariableInitialized" input: "bert/embeddings/word_embeddings/adam_v" attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_208" op: "IsVariableInitialized" input: "bert/embeddings/token_type_embeddings/adam_m" attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_209" op: "IsVariableInitialized" input: "bert/embeddings/token_type_embeddings/adam_v" attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_210" op: "IsVariableInitialized" input: "bert/embeddings/position_embeddings/adam_m" attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_211" op: "IsVariableInitialized" input: "bert/embeddings/position_embeddings/adam_v" attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_212" op: "IsVariableInitialized" input: "bert/embeddings/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_213" op: "IsVariableInitialized" input: "bert/embeddings/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_214" op: "IsVariableInitialized" input: "bert/embeddings/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_215" op: "IsVariableInitialized" input: "bert/embeddings/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_216" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_217" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_218" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/self/query/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_219" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/self/query/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_220" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_221" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_222" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/self/key/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_223" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/self/key/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_224" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_225" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_226" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/self/value/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_227" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/self/value/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_228" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_229" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_230" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_231" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_232" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_233" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_234" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_235" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_236" op: "IsVariableInitialized" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_237" op: "IsVariableInitialized" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_238" op: "IsVariableInitialized" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_239" op: "IsVariableInitialized" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_240" op: "IsVariableInitialized" input: "bert/encoder/layer_0/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_241" op: "IsVariableInitialized" input: "bert/encoder/layer_0/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_242" op: "IsVariableInitialized" input: "bert/encoder/layer_0/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_243" op: "IsVariableInitialized" input: "bert/encoder/layer_0/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_244" op: "IsVariableInitialized" input: "bert/encoder/layer_0/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_245" op: "IsVariableInitialized" input: "bert/encoder/layer_0/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_246" op: "IsVariableInitialized" input: "bert/encoder/layer_0/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_247" op: "IsVariableInitialized" input: "bert/encoder/layer_0/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_248" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_249" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_250" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/self/query/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_251" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/self/query/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_252" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_253" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_254" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/self/key/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_255" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/self/key/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_256" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_257" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_258" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/self/value/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_259" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/self/value/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_260" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_261" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_262" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_263" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_264" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_265" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_266" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_267" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_268" op: "IsVariableInitialized" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_269" op: "IsVariableInitialized" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_270" op: "IsVariableInitialized" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_271" op: "IsVariableInitialized" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_272" op: "IsVariableInitialized" input: "bert/encoder/layer_1/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_273" op: "IsVariableInitialized" input: "bert/encoder/layer_1/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_274" op: "IsVariableInitialized" input: "bert/encoder/layer_1/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_275" op: "IsVariableInitialized" input: "bert/encoder/layer_1/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_276" op: "IsVariableInitialized" input: "bert/encoder/layer_1/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_277" op: "IsVariableInitialized" input: "bert/encoder/layer_1/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_278" op: "IsVariableInitialized" input: "bert/encoder/layer_1/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_279" op: "IsVariableInitialized" input: "bert/encoder/layer_1/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_280" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_281" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_282" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/self/query/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_283" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/self/query/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_284" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_285" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_286" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/self/key/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_287" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/self/key/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_288" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_289" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_290" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/self/value/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_291" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/self/value/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_292" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_293" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_294" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_295" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_296" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_297" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_298" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_299" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_300" op: "IsVariableInitialized" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_301" op: "IsVariableInitialized" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_302" op: "IsVariableInitialized" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_303" op: "IsVariableInitialized" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_304" op: "IsVariableInitialized" input: "bert/encoder/layer_2/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_305" op: "IsVariableInitialized" input: "bert/encoder/layer_2/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_306" op: "IsVariableInitialized" input: "bert/encoder/layer_2/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_307" op: "IsVariableInitialized" input: "bert/encoder/layer_2/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_308" op: "IsVariableInitialized" input: "bert/encoder/layer_2/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_309" op: "IsVariableInitialized" input: "bert/encoder/layer_2/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_310" op: "IsVariableInitialized" input: "bert/encoder/layer_2/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_311" op: "IsVariableInitialized" input: "bert/encoder/layer_2/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_312" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_313" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_314" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/self/query/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_315" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/self/query/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_316" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_317" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_318" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/self/key/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_319" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/self/key/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_320" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_321" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_322" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/self/value/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_323" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/self/value/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_324" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_325" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_326" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_327" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_328" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_329" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_330" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_331" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_332" op: "IsVariableInitialized" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_333" op: "IsVariableInitialized" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_334" op: "IsVariableInitialized" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_335" op: "IsVariableInitialized" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_336" op: "IsVariableInitialized" input: "bert/encoder/layer_3/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_337" op: "IsVariableInitialized" input: "bert/encoder/layer_3/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_338" op: "IsVariableInitialized" input: "bert/encoder/layer_3/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_339" op: "IsVariableInitialized" input: "bert/encoder/layer_3/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_340" op: "IsVariableInitialized" input: "bert/encoder/layer_3/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_341" op: "IsVariableInitialized" input: "bert/encoder/layer_3/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_342" op: "IsVariableInitialized" input: "bert/encoder/layer_3/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_343" op: "IsVariableInitialized" input: "bert/encoder/layer_3/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_344" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_345" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_346" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/self/query/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_347" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/self/query/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_348" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_349" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_350" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/self/key/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_351" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/self/key/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_352" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_353" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_354" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/self/value/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_355" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/self/value/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_356" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_357" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_358" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_359" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_360" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_361" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_362" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_363" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_364" op: "IsVariableInitialized" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_365" op: "IsVariableInitialized" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_366" op: "IsVariableInitialized" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_367" op: "IsVariableInitialized" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_368" op: "IsVariableInitialized" input: "bert/encoder/layer_4/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_369" op: "IsVariableInitialized" input: "bert/encoder/layer_4/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_370" op: "IsVariableInitialized" input: "bert/encoder/layer_4/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_371" op: "IsVariableInitialized" input: "bert/encoder/layer_4/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_372" op: "IsVariableInitialized" input: "bert/encoder/layer_4/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_373" op: "IsVariableInitialized" input: "bert/encoder/layer_4/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_374" op: "IsVariableInitialized" input: "bert/encoder/layer_4/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_375" op: "IsVariableInitialized" input: "bert/encoder/layer_4/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_376" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_377" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_378" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/self/query/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_379" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/self/query/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_380" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_381" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_382" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/self/key/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_383" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/self/key/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_384" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_385" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_386" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/self/value/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_387" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/self/value/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_388" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_389" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_390" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_391" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_392" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_393" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_394" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_395" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_396" op: "IsVariableInitialized" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_397" op: "IsVariableInitialized" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_398" op: "IsVariableInitialized" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_399" op: "IsVariableInitialized" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_400" op: "IsVariableInitialized" input: "bert/encoder/layer_5/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_401" op: "IsVariableInitialized" input: "bert/encoder/layer_5/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_402" op: "IsVariableInitialized" input: "bert/encoder/layer_5/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_403" op: "IsVariableInitialized" input: "bert/encoder/layer_5/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_404" op: "IsVariableInitialized" input: "bert/encoder/layer_5/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_405" op: "IsVariableInitialized" input: "bert/encoder/layer_5/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_406" op: "IsVariableInitialized" input: "bert/encoder/layer_5/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_407" op: "IsVariableInitialized" input: "bert/encoder/layer_5/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_408" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_409" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_410" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/self/query/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_411" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/self/query/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_412" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_413" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_414" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/self/key/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_415" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/self/key/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_416" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_417" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_418" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/self/value/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_419" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/self/value/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_420" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_421" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_422" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_423" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_424" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_425" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_426" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_427" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_428" op: "IsVariableInitialized" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_429" op: "IsVariableInitialized" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_430" op: "IsVariableInitialized" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_431" op: "IsVariableInitialized" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_432" op: "IsVariableInitialized" input: "bert/encoder/layer_6/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_433" op: "IsVariableInitialized" input: "bert/encoder/layer_6/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_434" op: "IsVariableInitialized" input: "bert/encoder/layer_6/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_435" op: "IsVariableInitialized" input: "bert/encoder/layer_6/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_436" op: "IsVariableInitialized" input: "bert/encoder/layer_6/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_437" op: "IsVariableInitialized" input: "bert/encoder/layer_6/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_438" op: "IsVariableInitialized" input: "bert/encoder/layer_6/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_439" op: "IsVariableInitialized" input: "bert/encoder/layer_6/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_440" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_441" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_442" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/self/query/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_443" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/self/query/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_444" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_445" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_446" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/self/key/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_447" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/self/key/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_448" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_449" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_450" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/self/value/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_451" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/self/value/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_452" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_453" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_454" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_455" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_456" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_457" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_458" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_459" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_460" op: "IsVariableInitialized" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_461" op: "IsVariableInitialized" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_462" op: "IsVariableInitialized" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_463" op: "IsVariableInitialized" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_464" op: "IsVariableInitialized" input: "bert/encoder/layer_7/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_465" op: "IsVariableInitialized" input: "bert/encoder/layer_7/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_466" op: "IsVariableInitialized" input: "bert/encoder/layer_7/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_467" op: "IsVariableInitialized" input: "bert/encoder/layer_7/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_468" op: "IsVariableInitialized" input: "bert/encoder/layer_7/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_469" op: "IsVariableInitialized" input: "bert/encoder/layer_7/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_470" op: "IsVariableInitialized" input: "bert/encoder/layer_7/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_471" op: "IsVariableInitialized" input: "bert/encoder/layer_7/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_472" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_473" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_474" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/self/query/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_475" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/self/query/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_476" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_477" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_478" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/self/key/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_479" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/self/key/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_480" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_481" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_482" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/self/value/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_483" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/self/value/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_484" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_485" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_486" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_487" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_488" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_489" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_490" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_491" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_492" op: "IsVariableInitialized" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_493" op: "IsVariableInitialized" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_494" op: "IsVariableInitialized" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_495" op: "IsVariableInitialized" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_496" op: "IsVariableInitialized" input: "bert/encoder/layer_8/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_497" op: "IsVariableInitialized" input: "bert/encoder/layer_8/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_498" op: "IsVariableInitialized" input: "bert/encoder/layer_8/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_499" op: "IsVariableInitialized" input: "bert/encoder/layer_8/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_500" op: "IsVariableInitialized" input: "bert/encoder/layer_8/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_501" op: "IsVariableInitialized" input: "bert/encoder/layer_8/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_502" op: "IsVariableInitialized" input: "bert/encoder/layer_8/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_503" op: "IsVariableInitialized" input: "bert/encoder/layer_8/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_504" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_505" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_506" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/self/query/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_507" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/self/query/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_508" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_509" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_510" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/self/key/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_511" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/self/key/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_512" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_513" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_514" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/self/value/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_515" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/self/value/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_516" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_517" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_518" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_519" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_520" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_521" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_522" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_523" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_524" op: "IsVariableInitialized" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_525" op: "IsVariableInitialized" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_526" op: "IsVariableInitialized" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_527" op: "IsVariableInitialized" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_528" op: "IsVariableInitialized" input: "bert/encoder/layer_9/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_529" op: "IsVariableInitialized" input: "bert/encoder/layer_9/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_530" op: "IsVariableInitialized" input: "bert/encoder/layer_9/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_531" op: "IsVariableInitialized" input: "bert/encoder/layer_9/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_532" op: "IsVariableInitialized" input: "bert/encoder/layer_9/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_533" op: "IsVariableInitialized" input: "bert/encoder/layer_9/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_534" op: "IsVariableInitialized" input: "bert/encoder/layer_9/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_535" op: "IsVariableInitialized" input: "bert/encoder/layer_9/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_536" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_537" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_538" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/self/query/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_539" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/self/query/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_540" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_541" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_542" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/self/key/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_543" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/self/key/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_544" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_545" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_546" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/self/value/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_547" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/self/value/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_548" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_549" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_550" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_551" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_552" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_553" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_554" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_555" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_556" op: "IsVariableInitialized" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_557" op: "IsVariableInitialized" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_558" op: "IsVariableInitialized" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_559" op: "IsVariableInitialized" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_560" op: "IsVariableInitialized" input: "bert/encoder/layer_10/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_561" op: "IsVariableInitialized" input: "bert/encoder/layer_10/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_562" op: "IsVariableInitialized" input: "bert/encoder/layer_10/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_563" op: "IsVariableInitialized" input: "bert/encoder/layer_10/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_564" op: "IsVariableInitialized" input: "bert/encoder/layer_10/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_565" op: "IsVariableInitialized" input: "bert/encoder/layer_10/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_566" op: "IsVariableInitialized" input: "bert/encoder/layer_10/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_567" op: "IsVariableInitialized" input: "bert/encoder/layer_10/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_568" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_569" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_570" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/self/query/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_571" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/self/query/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_572" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_573" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_574" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/self/key/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_575" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/self/key/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_576" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_577" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_578" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/self/value/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_579" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/self/value/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_580" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_581" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_582" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_583" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_584" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_585" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_586" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_587" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_588" op: "IsVariableInitialized" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_589" op: "IsVariableInitialized" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_590" op: "IsVariableInitialized" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_591" op: "IsVariableInitialized" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_592" op: "IsVariableInitialized" input: "bert/encoder/layer_11/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_593" op: "IsVariableInitialized" input: "bert/encoder/layer_11/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_594" op: "IsVariableInitialized" input: "bert/encoder/layer_11/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_595" op: "IsVariableInitialized" input: "bert/encoder/layer_11/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_596" op: "IsVariableInitialized" input: "bert/encoder/layer_11/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_597" op: "IsVariableInitialized" input: "bert/encoder/layer_11/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_598" op: "IsVariableInitialized" input: "bert/encoder/layer_11/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_599" op: "IsVariableInitialized" input: "bert/encoder/layer_11/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_600" op: "IsVariableInitialized" input: "bert/pooler/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_601" op: "IsVariableInitialized" input: "bert/pooler/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_602" op: "IsVariableInitialized" input: "bert/pooler/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_603" op: "IsVariableInitialized" input: "bert/pooler/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_604" op: "IsVariableInitialized" input: "cls/predictions/transform/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_605" op: "IsVariableInitialized" input: "cls/predictions/transform/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_606" op: "IsVariableInitialized" input: "cls/predictions/transform/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_607" op: "IsVariableInitialized" input: "cls/predictions/transform/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_608" op: "IsVariableInitialized" input: "cls/predictions/transform/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_609" op: "IsVariableInitialized" input: "cls/predictions/transform/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_610" op: "IsVariableInitialized" input: "cls/predictions/transform/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_611" op: "IsVariableInitialized" input: "cls/predictions/transform/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_612" op: "IsVariableInitialized" input: "cls/predictions/output_bias/adam_m" attr { key: "_class" value { list { s: "loc:@cls/predictions/output_bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_613" op: "IsVariableInitialized" input: "cls/predictions/output_bias/adam_v" attr { key: "_class" value { list { s: "loc:@cls/predictions/output_bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_614" op: "IsVariableInitialized" input: "cls/seq_relationship/output_weights/adam_m" attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_weights/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_615" op: "IsVariableInitialized" input: "cls/seq_relationship/output_weights/adam_v" attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_weights/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_616" op: "IsVariableInitialized" input: "cls/seq_relationship/output_bias/adam_m" attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/IsVariableInitialized_617" op: "IsVariableInitialized" input: "cls/seq_relationship/output_bias/adam_v" attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables/stack" op: "Pack" input: "report_uninitialized_variables/VarIsInitializedOp" input: "report_uninitialized_variables/IsVariableInitialized" input: "report_uninitialized_variables/IsVariableInitialized_1" input: "report_uninitialized_variables/IsVariableInitialized_2" input: "report_uninitialized_variables/IsVariableInitialized_3" input: "report_uninitialized_variables/IsVariableInitialized_4" input: "report_uninitialized_variables/IsVariableInitialized_5" input: "report_uninitialized_variables/IsVariableInitialized_6" input: "report_uninitialized_variables/IsVariableInitialized_7" input: "report_uninitialized_variables/IsVariableInitialized_8" input: "report_uninitialized_variables/IsVariableInitialized_9" input: "report_uninitialized_variables/IsVariableInitialized_10" input: "report_uninitialized_variables/IsVariableInitialized_11" input: "report_uninitialized_variables/IsVariableInitialized_12" input: "report_uninitialized_variables/IsVariableInitialized_13" input: "report_uninitialized_variables/IsVariableInitialized_14" input: "report_uninitialized_variables/IsVariableInitialized_15" input: "report_uninitialized_variables/IsVariableInitialized_16" input: "report_uninitialized_variables/IsVariableInitialized_17" input: "report_uninitialized_variables/IsVariableInitialized_18" input: "report_uninitialized_variables/IsVariableInitialized_19" input: "report_uninitialized_variables/IsVariableInitialized_20" input: "report_uninitialized_variables/IsVariableInitialized_21" input: "report_uninitialized_variables/IsVariableInitialized_22" input: "report_uninitialized_variables/IsVariableInitialized_23" input: "report_uninitialized_variables/IsVariableInitialized_24" input: "report_uninitialized_variables/IsVariableInitialized_25" input: "report_uninitialized_variables/IsVariableInitialized_26" input: "report_uninitialized_variables/IsVariableInitialized_27" input: "report_uninitialized_variables/IsVariableInitialized_28" input: "report_uninitialized_variables/IsVariableInitialized_29" input: "report_uninitialized_variables/IsVariableInitialized_30" input: "report_uninitialized_variables/IsVariableInitialized_31" input: "report_uninitialized_variables/IsVariableInitialized_32" input: "report_uninitialized_variables/IsVariableInitialized_33" input: "report_uninitialized_variables/IsVariableInitialized_34" input: "report_uninitialized_variables/IsVariableInitialized_35" input: "report_uninitialized_variables/IsVariableInitialized_36" input: "report_uninitialized_variables/IsVariableInitialized_37" input: "report_uninitialized_variables/IsVariableInitialized_38" input: "report_uninitialized_variables/IsVariableInitialized_39" input: "report_uninitialized_variables/IsVariableInitialized_40" input: "report_uninitialized_variables/IsVariableInitialized_41" input: "report_uninitialized_variables/IsVariableInitialized_42" input: "report_uninitialized_variables/IsVariableInitialized_43" input: "report_uninitialized_variables/IsVariableInitialized_44" input: "report_uninitialized_variables/IsVariableInitialized_45" input: "report_uninitialized_variables/IsVariableInitialized_46" input: "report_uninitialized_variables/IsVariableInitialized_47" input: "report_uninitialized_variables/IsVariableInitialized_48" input: "report_uninitialized_variables/IsVariableInitialized_49" input: "report_uninitialized_variables/IsVariableInitialized_50" input: "report_uninitialized_variables/IsVariableInitialized_51" input: "report_uninitialized_variables/IsVariableInitialized_52" input: "report_uninitialized_variables/IsVariableInitialized_53" input: "report_uninitialized_variables/IsVariableInitialized_54" input: "report_uninitialized_variables/IsVariableInitialized_55" input: "report_uninitialized_variables/IsVariableInitialized_56" input: "report_uninitialized_variables/IsVariableInitialized_57" input: "report_uninitialized_variables/IsVariableInitialized_58" input: "report_uninitialized_variables/IsVariableInitialized_59" input: "report_uninitialized_variables/IsVariableInitialized_60" input: "report_uninitialized_variables/IsVariableInitialized_61" input: "report_uninitialized_variables/IsVariableInitialized_62" input: "report_uninitialized_variables/IsVariableInitialized_63" input: "report_uninitialized_variables/IsVariableInitialized_64" input: "report_uninitialized_variables/IsVariableInitialized_65" input: "report_uninitialized_variables/IsVariableInitialized_66" input: "report_uninitialized_variables/IsVariableInitialized_67" input: "report_uninitialized_variables/IsVariableInitialized_68" input: "report_uninitialized_variables/IsVariableInitialized_69" input: "report_uninitialized_variables/IsVariableInitialized_70" input: "report_uninitialized_variables/IsVariableInitialized_71" input: "report_uninitialized_variables/IsVariableInitialized_72" input: "report_uninitialized_variables/IsVariableInitialized_73" input: "report_uninitialized_variables/IsVariableInitialized_74" input: "report_uninitialized_variables/IsVariableInitialized_75" input: "report_uninitialized_variables/IsVariableInitialized_76" input: "report_uninitialized_variables/IsVariableInitialized_77" input: "report_uninitialized_variables/IsVariableInitialized_78" input: "report_uninitialized_variables/IsVariableInitialized_79" input: "report_uninitialized_variables/IsVariableInitialized_80" input: "report_uninitialized_variables/IsVariableInitialized_81" input: "report_uninitialized_variables/IsVariableInitialized_82" input: "report_uninitialized_variables/IsVariableInitialized_83" input: "report_uninitialized_variables/IsVariableInitialized_84" input: "report_uninitialized_variables/IsVariableInitialized_85" input: "report_uninitialized_variables/IsVariableInitialized_86" input: "report_uninitialized_variables/IsVariableInitialized_87" input: "report_uninitialized_variables/IsVariableInitialized_88" input: "report_uninitialized_variables/IsVariableInitialized_89" input: "report_uninitialized_variables/IsVariableInitialized_90" input: "report_uninitialized_variables/IsVariableInitialized_91" input: "report_uninitialized_variables/IsVariableInitialized_92" input: "report_uninitialized_variables/IsVariableInitialized_93" input: "report_uninitialized_variables/IsVariableInitialized_94" input: "report_uninitialized_variables/IsVariableInitialized_95" input: "report_uninitialized_variables/IsVariableInitialized_96" input: "report_uninitialized_variables/IsVariableInitialized_97" input: "report_uninitialized_variables/IsVariableInitialized_98" input: "report_uninitialized_variables/IsVariableInitialized_99" input: "report_uninitialized_variables/IsVariableInitialized_100" input: "report_uninitialized_variables/IsVariableInitialized_101" input: "report_uninitialized_variables/IsVariableInitialized_102" input: "report_uninitialized_variables/IsVariableInitialized_103" input: "report_uninitialized_variables/IsVariableInitialized_104" input: "report_uninitialized_variables/IsVariableInitialized_105" input: "report_uninitialized_variables/IsVariableInitialized_106" input: "report_uninitialized_variables/IsVariableInitialized_107" input: "report_uninitialized_variables/IsVariableInitialized_108" input: "report_uninitialized_variables/IsVariableInitialized_109" input: "report_uninitialized_variables/IsVariableInitialized_110" input: "report_uninitialized_variables/IsVariableInitialized_111" input: "report_uninitialized_variables/IsVariableInitialized_112" input: "report_uninitialized_variables/IsVariableInitialized_113" input: "report_uninitialized_variables/IsVariableInitialized_114" input: "report_uninitialized_variables/IsVariableInitialized_115" input: "report_uninitialized_variables/IsVariableInitialized_116" input: "report_uninitialized_variables/IsVariableInitialized_117" input: "report_uninitialized_variables/IsVariableInitialized_118" input: "report_uninitialized_variables/IsVariableInitialized_119" input: "report_uninitialized_variables/IsVariableInitialized_120" input: "report_uninitialized_variables/IsVariableInitialized_121" input: "report_uninitialized_variables/IsVariableInitialized_122" input: "report_uninitialized_variables/IsVariableInitialized_123" input: "report_uninitialized_variables/IsVariableInitialized_124" input: "report_uninitialized_variables/IsVariableInitialized_125" input: "report_uninitialized_variables/IsVariableInitialized_126" input: "report_uninitialized_variables/IsVariableInitialized_127" input: "report_uninitialized_variables/IsVariableInitialized_128" input: "report_uninitialized_variables/IsVariableInitialized_129" input: "report_uninitialized_variables/IsVariableInitialized_130" input: "report_uninitialized_variables/IsVariableInitialized_131" input: "report_uninitialized_variables/IsVariableInitialized_132" input: "report_uninitialized_variables/IsVariableInitialized_133" input: "report_uninitialized_variables/IsVariableInitialized_134" input: "report_uninitialized_variables/IsVariableInitialized_135" input: "report_uninitialized_variables/IsVariableInitialized_136" input: "report_uninitialized_variables/IsVariableInitialized_137" input: "report_uninitialized_variables/IsVariableInitialized_138" input: "report_uninitialized_variables/IsVariableInitialized_139" input: "report_uninitialized_variables/IsVariableInitialized_140" input: "report_uninitialized_variables/IsVariableInitialized_141" input: "report_uninitialized_variables/IsVariableInitialized_142" input: "report_uninitialized_variables/IsVariableInitialized_143" input: "report_uninitialized_variables/IsVariableInitialized_144" input: "report_uninitialized_variables/IsVariableInitialized_145" input: "report_uninitialized_variables/IsVariableInitialized_146" input: "report_uninitialized_variables/IsVariableInitialized_147" input: "report_uninitialized_variables/IsVariableInitialized_148" input: "report_uninitialized_variables/IsVariableInitialized_149" input: "report_uninitialized_variables/IsVariableInitialized_150" input: "report_uninitialized_variables/IsVariableInitialized_151" input: "report_uninitialized_variables/IsVariableInitialized_152" input: "report_uninitialized_variables/IsVariableInitialized_153" input: "report_uninitialized_variables/IsVariableInitialized_154" input: "report_uninitialized_variables/IsVariableInitialized_155" input: "report_uninitialized_variables/IsVariableInitialized_156" input: "report_uninitialized_variables/IsVariableInitialized_157" input: "report_uninitialized_variables/IsVariableInitialized_158" input: "report_uninitialized_variables/IsVariableInitialized_159" input: "report_uninitialized_variables/IsVariableInitialized_160" input: "report_uninitialized_variables/IsVariableInitialized_161" input: "report_uninitialized_variables/IsVariableInitialized_162" input: "report_uninitialized_variables/IsVariableInitialized_163" input: "report_uninitialized_variables/IsVariableInitialized_164" input: "report_uninitialized_variables/IsVariableInitialized_165" input: "report_uninitialized_variables/IsVariableInitialized_166" input: "report_uninitialized_variables/IsVariableInitialized_167" input: "report_uninitialized_variables/IsVariableInitialized_168" input: "report_uninitialized_variables/IsVariableInitialized_169" input: "report_uninitialized_variables/IsVariableInitialized_170" input: "report_uninitialized_variables/IsVariableInitialized_171" input: "report_uninitialized_variables/IsVariableInitialized_172" input: "report_uninitialized_variables/IsVariableInitialized_173" input: "report_uninitialized_variables/IsVariableInitialized_174" input: "report_uninitialized_variables/IsVariableInitialized_175" input: "report_uninitialized_variables/IsVariableInitialized_176" input: "report_uninitialized_variables/IsVariableInitialized_177" input: "report_uninitialized_variables/IsVariableInitialized_178" input: "report_uninitialized_variables/IsVariableInitialized_179" input: "report_uninitialized_variables/IsVariableInitialized_180" input: "report_uninitialized_variables/IsVariableInitialized_181" input: "report_uninitialized_variables/IsVariableInitialized_182" input: "report_uninitialized_variables/IsVariableInitialized_183" input: "report_uninitialized_variables/IsVariableInitialized_184" input: "report_uninitialized_variables/IsVariableInitialized_185" input: "report_uninitialized_variables/IsVariableInitialized_186" input: "report_uninitialized_variables/IsVariableInitialized_187" input: "report_uninitialized_variables/IsVariableInitialized_188" input: "report_uninitialized_variables/IsVariableInitialized_189" input: "report_uninitialized_variables/IsVariableInitialized_190" input: "report_uninitialized_variables/IsVariableInitialized_191" input: "report_uninitialized_variables/IsVariableInitialized_192" input: "report_uninitialized_variables/IsVariableInitialized_193" input: "report_uninitialized_variables/IsVariableInitialized_194" input: "report_uninitialized_variables/IsVariableInitialized_195" input: "report_uninitialized_variables/IsVariableInitialized_196" input: "report_uninitialized_variables/IsVariableInitialized_197" input: "report_uninitialized_variables/IsVariableInitialized_198" input: "report_uninitialized_variables/IsVariableInitialized_199" input: "report_uninitialized_variables/IsVariableInitialized_200" input: "report_uninitialized_variables/IsVariableInitialized_201" input: "report_uninitialized_variables/IsVariableInitialized_202" input: "report_uninitialized_variables/IsVariableInitialized_203" input: "report_uninitialized_variables/IsVariableInitialized_204" input: "report_uninitialized_variables/IsVariableInitialized_205" input: "report_uninitialized_variables/IsVariableInitialized_206" input: "report_uninitialized_variables/IsVariableInitialized_207" input: "report_uninitialized_variables/IsVariableInitialized_208" input: "report_uninitialized_variables/IsVariableInitialized_209" input: "report_uninitialized_variables/IsVariableInitialized_210" input: "report_uninitialized_variables/IsVariableInitialized_211" input: "report_uninitialized_variables/IsVariableInitialized_212" input: "report_uninitialized_variables/IsVariableInitialized_213" input: "report_uninitialized_variables/IsVariableInitialized_214" input: "report_uninitialized_variables/IsVariableInitialized_215" input: "report_uninitialized_variables/IsVariableInitialized_216" input: "report_uninitialized_variables/IsVariableInitialized_217" input: "report_uninitialized_variables/IsVariableInitialized_218" input: "report_uninitialized_variables/IsVariableInitialized_219" input: "report_uninitialized_variables/IsVariableInitialized_220" input: "report_uninitialized_variables/IsVariableInitialized_221" input: "report_uninitialized_variables/IsVariableInitialized_222" input: "report_uninitialized_variables/IsVariableInitialized_223" input: "report_uninitialized_variables/IsVariableInitialized_224" input: "report_uninitialized_variables/IsVariableInitialized_225" input: "report_uninitialized_variables/IsVariableInitialized_226" input: "report_uninitialized_variables/IsVariableInitialized_227" input: "report_uninitialized_variables/IsVariableInitialized_228" input: "report_uninitialized_variables/IsVariableInitialized_229" input: "report_uninitialized_variables/IsVariableInitialized_230" input: "report_uninitialized_variables/IsVariableInitialized_231" input: "report_uninitialized_variables/IsVariableInitialized_232" input: "report_uninitialized_variables/IsVariableInitialized_233" input: "report_uninitialized_variables/IsVariableInitialized_234" input: "report_uninitialized_variables/IsVariableInitialized_235" input: "report_uninitialized_variables/IsVariableInitialized_236" input: "report_uninitialized_variables/IsVariableInitialized_237" input: "report_uninitialized_variables/IsVariableInitialized_238" input: "report_uninitialized_variables/IsVariableInitialized_239" input: "report_uninitialized_variables/IsVariableInitialized_240" input: "report_uninitialized_variables/IsVariableInitialized_241" input: "report_uninitialized_variables/IsVariableInitialized_242" input: "report_uninitialized_variables/IsVariableInitialized_243" input: "report_uninitialized_variables/IsVariableInitialized_244" input: "report_uninitialized_variables/IsVariableInitialized_245" input: "report_uninitialized_variables/IsVariableInitialized_246" input: "report_uninitialized_variables/IsVariableInitialized_247" input: "report_uninitialized_variables/IsVariableInitialized_248" input: "report_uninitialized_variables/IsVariableInitialized_249" input: "report_uninitialized_variables/IsVariableInitialized_250" input: "report_uninitialized_variables/IsVariableInitialized_251" input: "report_uninitialized_variables/IsVariableInitialized_252" input: "report_uninitialized_variables/IsVariableInitialized_253" input: "report_uninitialized_variables/IsVariableInitialized_254" input: "report_uninitialized_variables/IsVariableInitialized_255" input: "report_uninitialized_variables/IsVariableInitialized_256" input: "report_uninitialized_variables/IsVariableInitialized_257" input: "report_uninitialized_variables/IsVariableInitialized_258" input: "report_uninitialized_variables/IsVariableInitialized_259" input: "report_uninitialized_variables/IsVariableInitialized_260" input: "report_uninitialized_variables/IsVariableInitialized_261" input: "report_uninitialized_variables/IsVariableInitialized_262" input: "report_uninitialized_variables/IsVariableInitialized_263" input: "report_uninitialized_variables/IsVariableInitialized_264" input: "report_uninitialized_variables/IsVariableInitialized_265" input: "report_uninitialized_variables/IsVariableInitialized_266" input: "report_uninitialized_variables/IsVariableInitialized_267" input: "report_uninitialized_variables/IsVariableInitialized_268" input: "report_uninitialized_variables/IsVariableInitialized_269" input: "report_uninitialized_variables/IsVariableInitialized_270" input: "report_uninitialized_variables/IsVariableInitialized_271" input: "report_uninitialized_variables/IsVariableInitialized_272" input: "report_uninitialized_variables/IsVariableInitialized_273" input: "report_uninitialized_variables/IsVariableInitialized_274" input: "report_uninitialized_variables/IsVariableInitialized_275" input: "report_uninitialized_variables/IsVariableInitialized_276" input: "report_uninitialized_variables/IsVariableInitialized_277" input: "report_uninitialized_variables/IsVariableInitialized_278" input: "report_uninitialized_variables/IsVariableInitialized_279" input: "report_uninitialized_variables/IsVariableInitialized_280" input: "report_uninitialized_variables/IsVariableInitialized_281" input: "report_uninitialized_variables/IsVariableInitialized_282" input: "report_uninitialized_variables/IsVariableInitialized_283" input: "report_uninitialized_variables/IsVariableInitialized_284" input: "report_uninitialized_variables/IsVariableInitialized_285" input: "report_uninitialized_variables/IsVariableInitialized_286" input: "report_uninitialized_variables/IsVariableInitialized_287" input: "report_uninitialized_variables/IsVariableInitialized_288" input: "report_uninitialized_variables/IsVariableInitialized_289" input: "report_uninitialized_variables/IsVariableInitialized_290" input: "report_uninitialized_variables/IsVariableInitialized_291" input: "report_uninitialized_variables/IsVariableInitialized_292" input: "report_uninitialized_variables/IsVariableInitialized_293" input: "report_uninitialized_variables/IsVariableInitialized_294" input: "report_uninitialized_variables/IsVariableInitialized_295" input: "report_uninitialized_variables/IsVariableInitialized_296" input: "report_uninitialized_variables/IsVariableInitialized_297" input: "report_uninitialized_variables/IsVariableInitialized_298" input: "report_uninitialized_variables/IsVariableInitialized_299" input: "report_uninitialized_variables/IsVariableInitialized_300" input: "report_uninitialized_variables/IsVariableInitialized_301" input: "report_uninitialized_variables/IsVariableInitialized_302" input: "report_uninitialized_variables/IsVariableInitialized_303" input: "report_uninitialized_variables/IsVariableInitialized_304" input: "report_uninitialized_variables/IsVariableInitialized_305" input: "report_uninitialized_variables/IsVariableInitialized_306" input: "report_uninitialized_variables/IsVariableInitialized_307" input: "report_uninitialized_variables/IsVariableInitialized_308" input: "report_uninitialized_variables/IsVariableInitialized_309" input: "report_uninitialized_variables/IsVariableInitialized_310" input: "report_uninitialized_variables/IsVariableInitialized_311" input: "report_uninitialized_variables/IsVariableInitialized_312" input: "report_uninitialized_variables/IsVariableInitialized_313" input: "report_uninitialized_variables/IsVariableInitialized_314" input: "report_uninitialized_variables/IsVariableInitialized_315" input: "report_uninitialized_variables/IsVariableInitialized_316" input: "report_uninitialized_variables/IsVariableInitialized_317" input: "report_uninitialized_variables/IsVariableInitialized_318" input: "report_uninitialized_variables/IsVariableInitialized_319" input: "report_uninitialized_variables/IsVariableInitialized_320" input: "report_uninitialized_variables/IsVariableInitialized_321" input: "report_uninitialized_variables/IsVariableInitialized_322" input: "report_uninitialized_variables/IsVariableInitialized_323" input: "report_uninitialized_variables/IsVariableInitialized_324" input: "report_uninitialized_variables/IsVariableInitialized_325" input: "report_uninitialized_variables/IsVariableInitialized_326" input: "report_uninitialized_variables/IsVariableInitialized_327" input: "report_uninitialized_variables/IsVariableInitialized_328" input: "report_uninitialized_variables/IsVariableInitialized_329" input: "report_uninitialized_variables/IsVariableInitialized_330" input: "report_uninitialized_variables/IsVariableInitialized_331" input: "report_uninitialized_variables/IsVariableInitialized_332" input: "report_uninitialized_variables/IsVariableInitialized_333" input: "report_uninitialized_variables/IsVariableInitialized_334" input: "report_uninitialized_variables/IsVariableInitialized_335" input: "report_uninitialized_variables/IsVariableInitialized_336" input: "report_uninitialized_variables/IsVariableInitialized_337" input: "report_uninitialized_variables/IsVariableInitialized_338" input: "report_uninitialized_variables/IsVariableInitialized_339" input: "report_uninitialized_variables/IsVariableInitialized_340" input: "report_uninitialized_variables/IsVariableInitialized_341" input: "report_uninitialized_variables/IsVariableInitialized_342" input: "report_uninitialized_variables/IsVariableInitialized_343" input: "report_uninitialized_variables/IsVariableInitialized_344" input: "report_uninitialized_variables/IsVariableInitialized_345" input: "report_uninitialized_variables/IsVariableInitialized_346" input: "report_uninitialized_variables/IsVariableInitialized_347" input: "report_uninitialized_variables/IsVariableInitialized_348" input: "report_uninitialized_variables/IsVariableInitialized_349" input: "report_uninitialized_variables/IsVariableInitialized_350" input: "report_uninitialized_variables/IsVariableInitialized_351" input: "report_uninitialized_variables/IsVariableInitialized_352" input: "report_uninitialized_variables/IsVariableInitialized_353" input: "report_uninitialized_variables/IsVariableInitialized_354" input: "report_uninitialized_variables/IsVariableInitialized_355" input: "report_uninitialized_variables/IsVariableInitialized_356" input: "report_uninitialized_variables/IsVariableInitialized_357" input: "report_uninitialized_variables/IsVariableInitialized_358" input: "report_uninitialized_variables/IsVariableInitialized_359" input: "report_uninitialized_variables/IsVariableInitialized_360" input: "report_uninitialized_variables/IsVariableInitialized_361" input: "report_uninitialized_variables/IsVariableInitialized_362" input: "report_uninitialized_variables/IsVariableInitialized_363" input: "report_uninitialized_variables/IsVariableInitialized_364" input: "report_uninitialized_variables/IsVariableInitialized_365" input: "report_uninitialized_variables/IsVariableInitialized_366" input: "report_uninitialized_variables/IsVariableInitialized_367" input: "report_uninitialized_variables/IsVariableInitialized_368" input: "report_uninitialized_variables/IsVariableInitialized_369" input: "report_uninitialized_variables/IsVariableInitialized_370" input: "report_uninitialized_variables/IsVariableInitialized_371" input: "report_uninitialized_variables/IsVariableInitialized_372" input: "report_uninitialized_variables/IsVariableInitialized_373" input: "report_uninitialized_variables/IsVariableInitialized_374" input: "report_uninitialized_variables/IsVariableInitialized_375" input: "report_uninitialized_variables/IsVariableInitialized_376" input: "report_uninitialized_variables/IsVariableInitialized_377" input: "report_uninitialized_variables/IsVariableInitialized_378" input: "report_uninitialized_variables/IsVariableInitialized_379" input: "report_uninitialized_variables/IsVariableInitialized_380" input: "report_uninitialized_variables/IsVariableInitialized_381" input: "report_uninitialized_variables/IsVariableInitialized_382" input: "report_uninitialized_variables/IsVariableInitialized_383" input: "report_uninitialized_variables/IsVariableInitialized_384" input: "report_uninitialized_variables/IsVariableInitialized_385" input: "report_uninitialized_variables/IsVariableInitialized_386" input: "report_uninitialized_variables/IsVariableInitialized_387" input: "report_uninitialized_variables/IsVariableInitialized_388" input: "report_uninitialized_variables/IsVariableInitialized_389" input: "report_uninitialized_variables/IsVariableInitialized_390" input: "report_uninitialized_variables/IsVariableInitialized_391" input: "report_uninitialized_variables/IsVariableInitialized_392" input: "report_uninitialized_variables/IsVariableInitialized_393" input: "report_uninitialized_variables/IsVariableInitialized_394" input: "report_uninitialized_variables/IsVariableInitialized_395" input: "report_uninitialized_variables/IsVariableInitialized_396" input: "report_uninitialized_variables/IsVariableInitialized_397" input: "report_uninitialized_variables/IsVariableInitialized_398" input: "report_uninitialized_variables/IsVariableInitialized_399" input: "report_uninitialized_variables/IsVariableInitialized_400" input: "report_uninitialized_variables/IsVariableInitialized_401" input: "report_uninitialized_variables/IsVariableInitialized_402" input: "report_uninitialized_variables/IsVariableInitialized_403" input: "report_uninitialized_variables/IsVariableInitialized_404" input: "report_uninitialized_variables/IsVariableInitialized_405" input: "report_uninitialized_variables/IsVariableInitialized_406" input: "report_uninitialized_variables/IsVariableInitialized_407" input: "report_uninitialized_variables/IsVariableInitialized_408" input: "report_uninitialized_variables/IsVariableInitialized_409" input: "report_uninitialized_variables/IsVariableInitialized_410" input: "report_uninitialized_variables/IsVariableInitialized_411" input: "report_uninitialized_variables/IsVariableInitialized_412" input: "report_uninitialized_variables/IsVariableInitialized_413" input: "report_uninitialized_variables/IsVariableInitialized_414" input: "report_uninitialized_variables/IsVariableInitialized_415" input: "report_uninitialized_variables/IsVariableInitialized_416" input: "report_uninitialized_variables/IsVariableInitialized_417" input: "report_uninitialized_variables/IsVariableInitialized_418" input: "report_uninitialized_variables/IsVariableInitialized_419" input: "report_uninitialized_variables/IsVariableInitialized_420" input: "report_uninitialized_variables/IsVariableInitialized_421" input: "report_uninitialized_variables/IsVariableInitialized_422" input: "report_uninitialized_variables/IsVariableInitialized_423" input: "report_uninitialized_variables/IsVariableInitialized_424" input: "report_uninitialized_variables/IsVariableInitialized_425" input: "report_uninitialized_variables/IsVariableInitialized_426" input: "report_uninitialized_variables/IsVariableInitialized_427" input: "report_uninitialized_variables/IsVariableInitialized_428" input: "report_uninitialized_variables/IsVariableInitialized_429" input: "report_uninitialized_variables/IsVariableInitialized_430" input: "report_uninitialized_variables/IsVariableInitialized_431" input: "report_uninitialized_variables/IsVariableInitialized_432" input: "report_uninitialized_variables/IsVariableInitialized_433" input: "report_uninitialized_variables/IsVariableInitialized_434" input: "report_uninitialized_variables/IsVariableInitialized_435" input: "report_uninitialized_variables/IsVariableInitialized_436" input: "report_uninitialized_variables/IsVariableInitialized_437" input: "report_uninitialized_variables/IsVariableInitialized_438" input: "report_uninitialized_variables/IsVariableInitialized_439" input: "report_uninitialized_variables/IsVariableInitialized_440" input: "report_uninitialized_variables/IsVariableInitialized_441" input: "report_uninitialized_variables/IsVariableInitialized_442" input: "report_uninitialized_variables/IsVariableInitialized_443" input: "report_uninitialized_variables/IsVariableInitialized_444" input: "report_uninitialized_variables/IsVariableInitialized_445" input: "report_uninitialized_variables/IsVariableInitialized_446" input: "report_uninitialized_variables/IsVariableInitialized_447" input: "report_uninitialized_variables/IsVariableInitialized_448" input: "report_uninitialized_variables/IsVariableInitialized_449" input: "report_uninitialized_variables/IsVariableInitialized_450" input: "report_uninitialized_variables/IsVariableInitialized_451" input: "report_uninitialized_variables/IsVariableInitialized_452" input: "report_uninitialized_variables/IsVariableInitialized_453" input: "report_uninitialized_variables/IsVariableInitialized_454" input: "report_uninitialized_variables/IsVariableInitialized_455" input: "report_uninitialized_variables/IsVariableInitialized_456" input: "report_uninitialized_variables/IsVariableInitialized_457" input: "report_uninitialized_variables/IsVariableInitialized_458" input: "report_uninitialized_variables/IsVariableInitialized_459" input: "report_uninitialized_variables/IsVariableInitialized_460" input: "report_uninitialized_variables/IsVariableInitialized_461" input: "report_uninitialized_variables/IsVariableInitialized_462" input: "report_uninitialized_variables/IsVariableInitialized_463" input: "report_uninitialized_variables/IsVariableInitialized_464" input: "report_uninitialized_variables/IsVariableInitialized_465" input: "report_uninitialized_variables/IsVariableInitialized_466" input: "report_uninitialized_variables/IsVariableInitialized_467" input: "report_uninitialized_variables/IsVariableInitialized_468" input: "report_uninitialized_variables/IsVariableInitialized_469" input: "report_uninitialized_variables/IsVariableInitialized_470" input: "report_uninitialized_variables/IsVariableInitialized_471" input: "report_uninitialized_variables/IsVariableInitialized_472" input: "report_uninitialized_variables/IsVariableInitialized_473" input: "report_uninitialized_variables/IsVariableInitialized_474" input: "report_uninitialized_variables/IsVariableInitialized_475" input: "report_uninitialized_variables/IsVariableInitialized_476" input: "report_uninitialized_variables/IsVariableInitialized_477" input: "report_uninitialized_variables/IsVariableInitialized_478" input: "report_uninitialized_variables/IsVariableInitialized_479" input: "report_uninitialized_variables/IsVariableInitialized_480" input: "report_uninitialized_variables/IsVariableInitialized_481" input: "report_uninitialized_variables/IsVariableInitialized_482" input: "report_uninitialized_variables/IsVariableInitialized_483" input: "report_uninitialized_variables/IsVariableInitialized_484" input: "report_uninitialized_variables/IsVariableInitialized_485" input: "report_uninitialized_variables/IsVariableInitialized_486" input: "report_uninitialized_variables/IsVariableInitialized_487" input: "report_uninitialized_variables/IsVariableInitialized_488" input: "report_uninitialized_variables/IsVariableInitialized_489" input: "report_uninitialized_variables/IsVariableInitialized_490" input: "report_uninitialized_variables/IsVariableInitialized_491" input: "report_uninitialized_variables/IsVariableInitialized_492" input: "report_uninitialized_variables/IsVariableInitialized_493" input: "report_uninitialized_variables/IsVariableInitialized_494" input: "report_uninitialized_variables/IsVariableInitialized_495" input: "report_uninitialized_variables/IsVariableInitialized_496" input: "report_uninitialized_variables/IsVariableInitialized_497" input: "report_uninitialized_variables/IsVariableInitialized_498" input: "report_uninitialized_variables/IsVariableInitialized_499" input: "report_uninitialized_variables/IsVariableInitialized_500" input: "report_uninitialized_variables/IsVariableInitialized_501" input: "report_uninitialized_variables/IsVariableInitialized_502" input: "report_uninitialized_variables/IsVariableInitialized_503" input: "report_uninitialized_variables/IsVariableInitialized_504" input: "report_uninitialized_variables/IsVariableInitialized_505" input: "report_uninitialized_variables/IsVariableInitialized_506" input: "report_uninitialized_variables/IsVariableInitialized_507" input: "report_uninitialized_variables/IsVariableInitialized_508" input: "report_uninitialized_variables/IsVariableInitialized_509" input: "report_uninitialized_variables/IsVariableInitialized_510" input: "report_uninitialized_variables/IsVariableInitialized_511" input: "report_uninitialized_variables/IsVariableInitialized_512" input: "report_uninitialized_variables/IsVariableInitialized_513" input: "report_uninitialized_variables/IsVariableInitialized_514" input: "report_uninitialized_variables/IsVariableInitialized_515" input: "report_uninitialized_variables/IsVariableInitialized_516" input: "report_uninitialized_variables/IsVariableInitialized_517" input: "report_uninitialized_variables/IsVariableInitialized_518" input: "report_uninitialized_variables/IsVariableInitialized_519" input: "report_uninitialized_variables/IsVariableInitialized_520" input: "report_uninitialized_variables/IsVariableInitialized_521" input: "report_uninitialized_variables/IsVariableInitialized_522" input: "report_uninitialized_variables/IsVariableInitialized_523" input: "report_uninitialized_variables/IsVariableInitialized_524" input: "report_uninitialized_variables/IsVariableInitialized_525" input: "report_uninitialized_variables/IsVariableInitialized_526" input: "report_uninitialized_variables/IsVariableInitialized_527" input: "report_uninitialized_variables/IsVariableInitialized_528" input: "report_uninitialized_variables/IsVariableInitialized_529" input: "report_uninitialized_variables/IsVariableInitialized_530" input: "report_uninitialized_variables/IsVariableInitialized_531" input: "report_uninitialized_variables/IsVariableInitialized_532" input: "report_uninitialized_variables/IsVariableInitialized_533" input: "report_uninitialized_variables/IsVariableInitialized_534" input: "report_uninitialized_variables/IsVariableInitialized_535" input: "report_uninitialized_variables/IsVariableInitialized_536" input: "report_uninitialized_variables/IsVariableInitialized_537" input: "report_uninitialized_variables/IsVariableInitialized_538" input: "report_uninitialized_variables/IsVariableInitialized_539" input: "report_uninitialized_variables/IsVariableInitialized_540" input: "report_uninitialized_variables/IsVariableInitialized_541" input: "report_uninitialized_variables/IsVariableInitialized_542" input: "report_uninitialized_variables/IsVariableInitialized_543" input: "report_uninitialized_variables/IsVariableInitialized_544" input: "report_uninitialized_variables/IsVariableInitialized_545" input: "report_uninitialized_variables/IsVariableInitialized_546" input: "report_uninitialized_variables/IsVariableInitialized_547" input: "report_uninitialized_variables/IsVariableInitialized_548" input: "report_uninitialized_variables/IsVariableInitialized_549" input: "report_uninitialized_variables/IsVariableInitialized_550" input: "report_uninitialized_variables/IsVariableInitialized_551" input: "report_uninitialized_variables/IsVariableInitialized_552" input: "report_uninitialized_variables/IsVariableInitialized_553" input: "report_uninitialized_variables/IsVariableInitialized_554" input: "report_uninitialized_variables/IsVariableInitialized_555" input: "report_uninitialized_variables/IsVariableInitialized_556" input: "report_uninitialized_variables/IsVariableInitialized_557" input: "report_uninitialized_variables/IsVariableInitialized_558" input: "report_uninitialized_variables/IsVariableInitialized_559" input: "report_uninitialized_variables/IsVariableInitialized_560" input: "report_uninitialized_variables/IsVariableInitialized_561" input: "report_uninitialized_variables/IsVariableInitialized_562" input: "report_uninitialized_variables/IsVariableInitialized_563" input: "report_uninitialized_variables/IsVariableInitialized_564" input: "report_uninitialized_variables/IsVariableInitialized_565" input: "report_uninitialized_variables/IsVariableInitialized_566" input: "report_uninitialized_variables/IsVariableInitialized_567" input: "report_uninitialized_variables/IsVariableInitialized_568" input: "report_uninitialized_variables/IsVariableInitialized_569" input: "report_uninitialized_variables/IsVariableInitialized_570" input: "report_uninitialized_variables/IsVariableInitialized_571" input: "report_uninitialized_variables/IsVariableInitialized_572" input: "report_uninitialized_variables/IsVariableInitialized_573" input: "report_uninitialized_variables/IsVariableInitialized_574" input: "report_uninitialized_variables/IsVariableInitialized_575" input: "report_uninitialized_variables/IsVariableInitialized_576" input: "report_uninitialized_variables/IsVariableInitialized_577" input: "report_uninitialized_variables/IsVariableInitialized_578" input: "report_uninitialized_variables/IsVariableInitialized_579" input: "report_uninitialized_variables/IsVariableInitialized_580" input: "report_uninitialized_variables/IsVariableInitialized_581" input: "report_uninitialized_variables/IsVariableInitialized_582" input: "report_uninitialized_variables/IsVariableInitialized_583" input: "report_uninitialized_variables/IsVariableInitialized_584" input: "report_uninitialized_variables/IsVariableInitialized_585" input: "report_uninitialized_variables/IsVariableInitialized_586" input: "report_uninitialized_variables/IsVariableInitialized_587" input: "report_uninitialized_variables/IsVariableInitialized_588" input: "report_uninitialized_variables/IsVariableInitialized_589" input: "report_uninitialized_variables/IsVariableInitialized_590" input: "report_uninitialized_variables/IsVariableInitialized_591" input: "report_uninitialized_variables/IsVariableInitialized_592" input: "report_uninitialized_variables/IsVariableInitialized_593" input: "report_uninitialized_variables/IsVariableInitialized_594" input: "report_uninitialized_variables/IsVariableInitialized_595" input: "report_uninitialized_variables/IsVariableInitialized_596" input: "report_uninitialized_variables/IsVariableInitialized_597" input: "report_uninitialized_variables/IsVariableInitialized_598" input: "report_uninitialized_variables/IsVariableInitialized_599" input: "report_uninitialized_variables/IsVariableInitialized_600" input: "report_uninitialized_variables/IsVariableInitialized_601" input: "report_uninitialized_variables/IsVariableInitialized_602" input: "report_uninitialized_variables/IsVariableInitialized_603" input: "report_uninitialized_variables/IsVariableInitialized_604" input: "report_uninitialized_variables/IsVariableInitialized_605" input: "report_uninitialized_variables/IsVariableInitialized_606" input: "report_uninitialized_variables/IsVariableInitialized_607" input: "report_uninitialized_variables/IsVariableInitialized_608" input: "report_uninitialized_variables/IsVariableInitialized_609" input: "report_uninitialized_variables/IsVariableInitialized_610" input: "report_uninitialized_variables/IsVariableInitialized_611" input: "report_uninitialized_variables/IsVariableInitialized_612" input: "report_uninitialized_variables/IsVariableInitialized_613" input: "report_uninitialized_variables/IsVariableInitialized_614" input: "report_uninitialized_variables/IsVariableInitialized_615" input: "report_uninitialized_variables/IsVariableInitialized_616" input: "report_uninitialized_variables/IsVariableInitialized_617" device: "/device:CPU:0" attr { key: "N" value { i: 619 } } attr { key: "T" value { type: DT_BOOL } } attr { key: "_output_shapes" value { list { shape { dim { size: 619 } } } } } attr { key: "axis" value { i: 0 } } } node { name: "report_uninitialized_variables/LogicalNot" op: "LogicalNot" input: "report_uninitialized_variables/stack" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 619 } } } } } } node { name: "report_uninitialized_variables/Const" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 619 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 619 } } string_val: "global_step" string_val: "bert/embeddings/word_embeddings" string_val: "bert/embeddings/token_type_embeddings" string_val: "bert/embeddings/position_embeddings" string_val: "bert/embeddings/LayerNorm/beta" string_val: "bert/embeddings/LayerNorm/gamma" string_val: "bert/encoder/layer_0/attention/self/query/kernel" string_val: "bert/encoder/layer_0/attention/self/query/bias" string_val: "bert/encoder/layer_0/attention/self/key/kernel" string_val: "bert/encoder/layer_0/attention/self/key/bias" string_val: "bert/encoder/layer_0/attention/self/value/kernel" string_val: "bert/encoder/layer_0/attention/self/value/bias" string_val: "bert/encoder/layer_0/attention/output/dense/kernel" string_val: "bert/encoder/layer_0/attention/output/dense/bias" string_val: "bert/encoder/layer_0/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_0/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_0/intermediate/dense/kernel" string_val: "bert/encoder/layer_0/intermediate/dense/bias" string_val: "bert/encoder/layer_0/output/dense/kernel" string_val: "bert/encoder/layer_0/output/dense/bias" string_val: "bert/encoder/layer_0/output/LayerNorm/beta" string_val: "bert/encoder/layer_0/output/LayerNorm/gamma" string_val: "bert/encoder/layer_1/attention/self/query/kernel" string_val: "bert/encoder/layer_1/attention/self/query/bias" string_val: "bert/encoder/layer_1/attention/self/key/kernel" string_val: "bert/encoder/layer_1/attention/self/key/bias" string_val: "bert/encoder/layer_1/attention/self/value/kernel" string_val: "bert/encoder/layer_1/attention/self/value/bias" string_val: "bert/encoder/layer_1/attention/output/dense/kernel" string_val: "bert/encoder/layer_1/attention/output/dense/bias" string_val: "bert/encoder/layer_1/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_1/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_1/intermediate/dense/kernel" string_val: "bert/encoder/layer_1/intermediate/dense/bias" string_val: "bert/encoder/layer_1/output/dense/kernel" string_val: "bert/encoder/layer_1/output/dense/bias" string_val: "bert/encoder/layer_1/output/LayerNorm/beta" string_val: "bert/encoder/layer_1/output/LayerNorm/gamma" string_val: "bert/encoder/layer_2/attention/self/query/kernel" string_val: "bert/encoder/layer_2/attention/self/query/bias" string_val: "bert/encoder/layer_2/attention/self/key/kernel" string_val: "bert/encoder/layer_2/attention/self/key/bias" string_val: "bert/encoder/layer_2/attention/self/value/kernel" string_val: "bert/encoder/layer_2/attention/self/value/bias" string_val: "bert/encoder/layer_2/attention/output/dense/kernel" string_val: "bert/encoder/layer_2/attention/output/dense/bias" string_val: "bert/encoder/layer_2/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_2/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_2/intermediate/dense/kernel" string_val: "bert/encoder/layer_2/intermediate/dense/bias" string_val: "bert/encoder/layer_2/output/dense/kernel" string_val: "bert/encoder/layer_2/output/dense/bias" string_val: "bert/encoder/layer_2/output/LayerNorm/beta" string_val: "bert/encoder/layer_2/output/LayerNorm/gamma" string_val: "bert/encoder/layer_3/attention/self/query/kernel" string_val: "bert/encoder/layer_3/attention/self/query/bias" string_val: "bert/encoder/layer_3/attention/self/key/kernel" string_val: "bert/encoder/layer_3/attention/self/key/bias" string_val: "bert/encoder/layer_3/attention/self/value/kernel" string_val: "bert/encoder/layer_3/attention/self/value/bias" string_val: "bert/encoder/layer_3/attention/output/dense/kernel" string_val: "bert/encoder/layer_3/attention/output/dense/bias" string_val: "bert/encoder/layer_3/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_3/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_3/intermediate/dense/kernel" string_val: "bert/encoder/layer_3/intermediate/dense/bias" string_val: "bert/encoder/layer_3/output/dense/kernel" string_val: "bert/encoder/layer_3/output/dense/bias" string_val: "bert/encoder/layer_3/output/LayerNorm/beta" string_val: "bert/encoder/layer_3/output/LayerNorm/gamma" string_val: "bert/encoder/layer_4/attention/self/query/kernel" string_val: "bert/encoder/layer_4/attention/self/query/bias" string_val: "bert/encoder/layer_4/attention/self/key/kernel" string_val: "bert/encoder/layer_4/attention/self/key/bias" string_val: "bert/encoder/layer_4/attention/self/value/kernel" string_val: "bert/encoder/layer_4/attention/self/value/bias" string_val: "bert/encoder/layer_4/attention/output/dense/kernel" string_val: "bert/encoder/layer_4/attention/output/dense/bias" string_val: "bert/encoder/layer_4/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_4/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_4/intermediate/dense/kernel" string_val: "bert/encoder/layer_4/intermediate/dense/bias" string_val: "bert/encoder/layer_4/output/dense/kernel" string_val: "bert/encoder/layer_4/output/dense/bias" string_val: "bert/encoder/layer_4/output/LayerNorm/beta" string_val: "bert/encoder/layer_4/output/LayerNorm/gamma" string_val: "bert/encoder/layer_5/attention/self/query/kernel" string_val: "bert/encoder/layer_5/attention/self/query/bias" string_val: "bert/encoder/layer_5/attention/self/key/kernel" string_val: "bert/encoder/layer_5/attention/self/key/bias" string_val: "bert/encoder/layer_5/attention/self/value/kernel" string_val: "bert/encoder/layer_5/attention/self/value/bias" string_val: "bert/encoder/layer_5/attention/output/dense/kernel" string_val: "bert/encoder/layer_5/attention/output/dense/bias" string_val: "bert/encoder/layer_5/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_5/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_5/intermediate/dense/kernel" string_val: "bert/encoder/layer_5/intermediate/dense/bias" string_val: "bert/encoder/layer_5/output/dense/kernel" string_val: "bert/encoder/layer_5/output/dense/bias" string_val: "bert/encoder/layer_5/output/LayerNorm/beta" string_val: "bert/encoder/layer_5/output/LayerNorm/gamma" string_val: "bert/encoder/layer_6/attention/self/query/kernel" string_val: "bert/encoder/layer_6/attention/self/query/bias" string_val: "bert/encoder/layer_6/attention/self/key/kernel" string_val: "bert/encoder/layer_6/attention/self/key/bias" string_val: "bert/encoder/layer_6/attention/self/value/kernel" string_val: "bert/encoder/layer_6/attention/self/value/bias" string_val: "bert/encoder/layer_6/attention/output/dense/kernel" string_val: "bert/encoder/layer_6/attention/output/dense/bias" string_val: "bert/encoder/layer_6/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_6/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_6/intermediate/dense/kernel" string_val: "bert/encoder/layer_6/intermediate/dense/bias" string_val: "bert/encoder/layer_6/output/dense/kernel" string_val: "bert/encoder/layer_6/output/dense/bias" string_val: "bert/encoder/layer_6/output/LayerNorm/beta" string_val: "bert/encoder/layer_6/output/LayerNorm/gamma" string_val: "bert/encoder/layer_7/attention/self/query/kernel" string_val: "bert/encoder/layer_7/attention/self/query/bias" string_val: "bert/encoder/layer_7/attention/self/key/kernel" string_val: "bert/encoder/layer_7/attention/self/key/bias" string_val: "bert/encoder/layer_7/attention/self/value/kernel" string_val: "bert/encoder/layer_7/attention/self/value/bias" string_val: "bert/encoder/layer_7/attention/output/dense/kernel" string_val: "bert/encoder/layer_7/attention/output/dense/bias" string_val: "bert/encoder/layer_7/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_7/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_7/intermediate/dense/kernel" string_val: "bert/encoder/layer_7/intermediate/dense/bias" string_val: "bert/encoder/layer_7/output/dense/kernel" string_val: "bert/encoder/layer_7/output/dense/bias" string_val: "bert/encoder/layer_7/output/LayerNorm/beta" string_val: "bert/encoder/layer_7/output/LayerNorm/gamma" string_val: "bert/encoder/layer_8/attention/self/query/kernel" string_val: "bert/encoder/layer_8/attention/self/query/bias" string_val: "bert/encoder/layer_8/attention/self/key/kernel" string_val: "bert/encoder/layer_8/attention/self/key/bias" string_val: "bert/encoder/layer_8/attention/self/value/kernel" string_val: "bert/encoder/layer_8/attention/self/value/bias" string_val: "bert/encoder/layer_8/attention/output/dense/kernel" string_val: "bert/encoder/layer_8/attention/output/dense/bias" string_val: "bert/encoder/layer_8/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_8/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_8/intermediate/dense/kernel" string_val: "bert/encoder/layer_8/intermediate/dense/bias" string_val: "bert/encoder/layer_8/output/dense/kernel" string_val: "bert/encoder/layer_8/output/dense/bias" string_val: "bert/encoder/layer_8/output/LayerNorm/beta" string_val: "bert/encoder/layer_8/output/LayerNorm/gamma" string_val: "bert/encoder/layer_9/attention/self/query/kernel" string_val: "bert/encoder/layer_9/attention/self/query/bias" string_val: "bert/encoder/layer_9/attention/self/key/kernel" string_val: "bert/encoder/layer_9/attention/self/key/bias" string_val: "bert/encoder/layer_9/attention/self/value/kernel" string_val: "bert/encoder/layer_9/attention/self/value/bias" string_val: "bert/encoder/layer_9/attention/output/dense/kernel" string_val: "bert/encoder/layer_9/attention/output/dense/bias" string_val: "bert/encoder/layer_9/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_9/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_9/intermediate/dense/kernel" string_val: "bert/encoder/layer_9/intermediate/dense/bias" string_val: "bert/encoder/layer_9/output/dense/kernel" string_val: "bert/encoder/layer_9/output/dense/bias" string_val: "bert/encoder/layer_9/output/LayerNorm/beta" string_val: "bert/encoder/layer_9/output/LayerNorm/gamma" string_val: "bert/encoder/layer_10/attention/self/query/kernel" string_val: "bert/encoder/layer_10/attention/self/query/bias" string_val: "bert/encoder/layer_10/attention/self/key/kernel" string_val: "bert/encoder/layer_10/attention/self/key/bias" string_val: "bert/encoder/layer_10/attention/self/value/kernel" string_val: "bert/encoder/layer_10/attention/self/value/bias" string_val: "bert/encoder/layer_10/attention/output/dense/kernel" string_val: "bert/encoder/layer_10/attention/output/dense/bias" string_val: "bert/encoder/layer_10/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_10/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_10/intermediate/dense/kernel" string_val: "bert/encoder/layer_10/intermediate/dense/bias" string_val: "bert/encoder/layer_10/output/dense/kernel" string_val: "bert/encoder/layer_10/output/dense/bias" string_val: "bert/encoder/layer_10/output/LayerNorm/beta" string_val: "bert/encoder/layer_10/output/LayerNorm/gamma" string_val: "bert/encoder/layer_11/attention/self/query/kernel" string_val: "bert/encoder/layer_11/attention/self/query/bias" string_val: "bert/encoder/layer_11/attention/self/key/kernel" string_val: "bert/encoder/layer_11/attention/self/key/bias" string_val: "bert/encoder/layer_11/attention/self/value/kernel" string_val: "bert/encoder/layer_11/attention/self/value/bias" string_val: "bert/encoder/layer_11/attention/output/dense/kernel" string_val: "bert/encoder/layer_11/attention/output/dense/bias" string_val: "bert/encoder/layer_11/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_11/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_11/intermediate/dense/kernel" string_val: "bert/encoder/layer_11/intermediate/dense/bias" string_val: "bert/encoder/layer_11/output/dense/kernel" string_val: "bert/encoder/layer_11/output/dense/bias" string_val: "bert/encoder/layer_11/output/LayerNorm/beta" string_val: "bert/encoder/layer_11/output/LayerNorm/gamma" string_val: "bert/pooler/dense/kernel" string_val: "bert/pooler/dense/bias" string_val: "cls/predictions/transform/dense/kernel" string_val: "cls/predictions/transform/dense/bias" string_val: "cls/predictions/transform/LayerNorm/beta" string_val: "cls/predictions/transform/LayerNorm/gamma" string_val: "cls/predictions/output_bias" string_val: "cls/seq_relationship/output_weights" string_val: "cls/seq_relationship/output_bias" string_val: "bert/embeddings/word_embeddings/adam_m" string_val: "bert/embeddings/word_embeddings/adam_v" string_val: "bert/embeddings/token_type_embeddings/adam_m" string_val: "bert/embeddings/token_type_embeddings/adam_v" string_val: "bert/embeddings/position_embeddings/adam_m" string_val: "bert/embeddings/position_embeddings/adam_v" string_val: "bert/embeddings/LayerNorm/beta/adam_m" string_val: "bert/embeddings/LayerNorm/beta/adam_v" string_val: "bert/embeddings/LayerNorm/gamma/adam_m" string_val: "bert/embeddings/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_0/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_0/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_0/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_0/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_0/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_0/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_0/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_0/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_0/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_0/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_0/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_0/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_0/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_0/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_0/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_0/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_0/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_0/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_0/output/dense/bias/adam_m" string_val: "bert/encoder/layer_0/output/dense/bias/adam_v" string_val: "bert/encoder/layer_0/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_0/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_0/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_0/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_1/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_1/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_1/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_1/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_1/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_1/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_1/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_1/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_1/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_1/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_1/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_1/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_1/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_1/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_1/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_1/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_1/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_1/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_1/output/dense/bias/adam_m" string_val: "bert/encoder/layer_1/output/dense/bias/adam_v" string_val: "bert/encoder/layer_1/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_1/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_1/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_1/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_2/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_2/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_2/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_2/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_2/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_2/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_2/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_2/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_2/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_2/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_2/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_2/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_2/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_2/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_2/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_2/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_2/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_2/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_2/output/dense/bias/adam_m" string_val: "bert/encoder/layer_2/output/dense/bias/adam_v" string_val: "bert/encoder/layer_2/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_2/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_2/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_2/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_3/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_3/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_3/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_3/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_3/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_3/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_3/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_3/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_3/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_3/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_3/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_3/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_3/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_3/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_3/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_3/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_3/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_3/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_3/output/dense/bias/adam_m" string_val: "bert/encoder/layer_3/output/dense/bias/adam_v" string_val: "bert/encoder/layer_3/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_3/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_3/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_3/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_4/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_4/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_4/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_4/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_4/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_4/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_4/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_4/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_4/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_4/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_4/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_4/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_4/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_4/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_4/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_4/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_4/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_4/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_4/output/dense/bias/adam_m" string_val: "bert/encoder/layer_4/output/dense/bias/adam_v" string_val: "bert/encoder/layer_4/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_4/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_4/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_4/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_5/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_5/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_5/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_5/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_5/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_5/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_5/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_5/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_5/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_5/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_5/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_5/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_5/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_5/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_5/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_5/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_5/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_5/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_5/output/dense/bias/adam_m" string_val: "bert/encoder/layer_5/output/dense/bias/adam_v" string_val: "bert/encoder/layer_5/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_5/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_5/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_5/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_6/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_6/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_6/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_6/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_6/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_6/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_6/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_6/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_6/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_6/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_6/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_6/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_6/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_6/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_6/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_6/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_6/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_6/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_6/output/dense/bias/adam_m" string_val: "bert/encoder/layer_6/output/dense/bias/adam_v" string_val: "bert/encoder/layer_6/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_6/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_6/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_6/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_7/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_7/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_7/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_7/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_7/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_7/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_7/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_7/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_7/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_7/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_7/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_7/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_7/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_7/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_7/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_7/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_7/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_7/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_7/output/dense/bias/adam_m" string_val: "bert/encoder/layer_7/output/dense/bias/adam_v" string_val: "bert/encoder/layer_7/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_7/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_7/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_7/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_8/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_8/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_8/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_8/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_8/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_8/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_8/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_8/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_8/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_8/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_8/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_8/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_8/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_8/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_8/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_8/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_8/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_8/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_8/output/dense/bias/adam_m" string_val: "bert/encoder/layer_8/output/dense/bias/adam_v" string_val: "bert/encoder/layer_8/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_8/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_8/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_8/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_9/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_9/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_9/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_9/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_9/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_9/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_9/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_9/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_9/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_9/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_9/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_9/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_9/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_9/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_9/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_9/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_9/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_9/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_9/output/dense/bias/adam_m" string_val: "bert/encoder/layer_9/output/dense/bias/adam_v" string_val: "bert/encoder/layer_9/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_9/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_9/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_9/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_10/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_10/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_10/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_10/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_10/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_10/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_10/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_10/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_10/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_10/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_10/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_10/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_10/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_10/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_10/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_10/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_10/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_10/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_10/output/dense/bias/adam_m" string_val: "bert/encoder/layer_10/output/dense/bias/adam_v" string_val: "bert/encoder/layer_10/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_10/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_10/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_10/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_11/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_11/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_11/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_11/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_11/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_11/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_11/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_11/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_11/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_11/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_11/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_11/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_11/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_11/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_11/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_11/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_11/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_11/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_11/output/dense/bias/adam_m" string_val: "bert/encoder/layer_11/output/dense/bias/adam_v" string_val: "bert/encoder/layer_11/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_11/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_11/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_11/output/LayerNorm/gamma/adam_v" string_val: "bert/pooler/dense/kernel/adam_m" string_val: "bert/pooler/dense/kernel/adam_v" string_val: "bert/pooler/dense/bias/adam_m" string_val: "bert/pooler/dense/bias/adam_v" string_val: "cls/predictions/transform/dense/kernel/adam_m" string_val: "cls/predictions/transform/dense/kernel/adam_v" string_val: "cls/predictions/transform/dense/bias/adam_m" string_val: "cls/predictions/transform/dense/bias/adam_v" string_val: "cls/predictions/transform/LayerNorm/beta/adam_m" string_val: "cls/predictions/transform/LayerNorm/beta/adam_v" string_val: "cls/predictions/transform/LayerNorm/gamma/adam_m" string_val: "cls/predictions/transform/LayerNorm/gamma/adam_v" string_val: "cls/predictions/output_bias/adam_m" string_val: "cls/predictions/output_bias/adam_v" string_val: "cls/seq_relationship/output_weights/adam_m" string_val: "cls/seq_relationship/output_weights/adam_v" string_val: "cls/seq_relationship/output_bias/adam_m" string_val: "cls/seq_relationship/output_bias/adam_v" } } } } node { name: "report_uninitialized_variables/boolean_mask/Shape" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 619 } } } } node { name: "report_uninitialized_variables/boolean_mask/strided_slice/stack" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "report_uninitialized_variables/boolean_mask/strided_slice/stack_1" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "report_uninitialized_variables/boolean_mask/strided_slice/stack_2" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "report_uninitialized_variables/boolean_mask/strided_slice" op: "StridedSlice" input: "report_uninitialized_variables/boolean_mask/Shape" input: "report_uninitialized_variables/boolean_mask/strided_slice/stack" input: "report_uninitialized_variables/boolean_mask/strided_slice/stack_1" input: "report_uninitialized_variables/boolean_mask/strided_slice/stack_2" device: "/device:CPU:0" attr { key: "Index" value { type: DT_INT32 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "begin_mask" value { i: 0 } } attr { key: "ellipsis_mask" value { i: 0 } } attr { key: "end_mask" value { i: 0 } } attr { key: "new_axis_mask" value { i: 0 } } attr { key: "shrink_axis_mask" value { i: 0 } } } node { name: "report_uninitialized_variables/boolean_mask/Prod/reduction_indices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "report_uninitialized_variables/boolean_mask/Prod" op: "Prod" input: "report_uninitialized_variables/boolean_mask/strided_slice" input: "report_uninitialized_variables/boolean_mask/Prod/reduction_indices" device: "/device:CPU:0" attr { key: "T" value { type: DT_INT32 } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "report_uninitialized_variables/boolean_mask/Shape_1" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 619 } } } } node { name: "report_uninitialized_variables/boolean_mask/strided_slice_1/stack" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "report_uninitialized_variables/boolean_mask/strided_slice_1/stack_1" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "report_uninitialized_variables/boolean_mask/strided_slice_1/stack_2" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "report_uninitialized_variables/boolean_mask/strided_slice_1" op: "StridedSlice" input: "report_uninitialized_variables/boolean_mask/Shape_1" input: "report_uninitialized_variables/boolean_mask/strided_slice_1/stack" input: "report_uninitialized_variables/boolean_mask/strided_slice_1/stack_1" input: "report_uninitialized_variables/boolean_mask/strided_slice_1/stack_2" device: "/device:CPU:0" attr { key: "Index" value { type: DT_INT32 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "begin_mask" value { i: 1 } } attr { key: "ellipsis_mask" value { i: 0 } } attr { key: "end_mask" value { i: 0 } } attr { key: "new_axis_mask" value { i: 0 } } attr { key: "shrink_axis_mask" value { i: 0 } } } node { name: "report_uninitialized_variables/boolean_mask/Shape_2" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 619 } } } } node { name: "report_uninitialized_variables/boolean_mask/strided_slice_2/stack" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "report_uninitialized_variables/boolean_mask/strided_slice_2/stack_1" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "report_uninitialized_variables/boolean_mask/strided_slice_2/stack_2" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "report_uninitialized_variables/boolean_mask/strided_slice_2" op: "StridedSlice" input: "report_uninitialized_variables/boolean_mask/Shape_2" input: "report_uninitialized_variables/boolean_mask/strided_slice_2/stack" input: "report_uninitialized_variables/boolean_mask/strided_slice_2/stack_1" input: "report_uninitialized_variables/boolean_mask/strided_slice_2/stack_2" device: "/device:CPU:0" attr { key: "Index" value { type: DT_INT32 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "begin_mask" value { i: 0 } } attr { key: "ellipsis_mask" value { i: 0 } } attr { key: "end_mask" value { i: 1 } } attr { key: "new_axis_mask" value { i: 0 } } attr { key: "shrink_axis_mask" value { i: 0 } } } node { name: "report_uninitialized_variables/boolean_mask/concat/values_1" op: "Pack" input: "report_uninitialized_variables/boolean_mask/Prod" device: "/device:CPU:0" attr { key: "N" value { i: 1 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "axis" value { i: 0 } } } node { name: "report_uninitialized_variables/boolean_mask/concat/axis" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "report_uninitialized_variables/boolean_mask/concat" op: "ConcatV2" input: "report_uninitialized_variables/boolean_mask/strided_slice_1" input: "report_uninitialized_variables/boolean_mask/concat/values_1" input: "report_uninitialized_variables/boolean_mask/strided_slice_2" input: "report_uninitialized_variables/boolean_mask/concat/axis" device: "/device:CPU:0" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "report_uninitialized_variables/boolean_mask/Reshape" op: "Reshape" input: "report_uninitialized_variables/Const" input: "report_uninitialized_variables/boolean_mask/concat" device: "/device:CPU:0" attr { key: "T" value { type: DT_STRING } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 619 } } } } } } node { name: "report_uninitialized_variables/boolean_mask/Reshape_1/shape" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: -1 } } } } node { name: "report_uninitialized_variables/boolean_mask/Reshape_1" op: "Reshape" input: "report_uninitialized_variables/LogicalNot" input: "report_uninitialized_variables/boolean_mask/Reshape_1/shape" device: "/device:CPU:0" attr { key: "T" value { type: DT_BOOL } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 619 } } } } } } node { name: "report_uninitialized_variables/boolean_mask/Where" op: "Where" input: "report_uninitialized_variables/boolean_mask/Reshape_1" device: "/device:CPU:0" attr { key: "T" value { type: DT_BOOL } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } dim { size: 1 } } } } } } node { name: "report_uninitialized_variables/boolean_mask/Squeeze" op: "Squeeze" input: "report_uninitialized_variables/boolean_mask/Where" device: "/device:CPU:0" attr { key: "T" value { type: DT_INT64 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } } } } attr { key: "squeeze_dims" value { list { i: 1 } } } } node { name: "report_uninitialized_variables/boolean_mask/GatherV2/axis" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "report_uninitialized_variables/boolean_mask/GatherV2" op: "GatherV2" input: "report_uninitialized_variables/boolean_mask/Reshape" input: "report_uninitialized_variables/boolean_mask/Squeeze" input: "report_uninitialized_variables/boolean_mask/GatherV2/axis" device: "/device:CPU:0" attr { key: "Taxis" value { type: DT_INT32 } } attr { key: "Tindices" value { type: DT_INT64 } } attr { key: "Tparams" value { type: DT_STRING } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } } } } } node { name: "report_uninitialized_resources/Const" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { } } } } } } node { name: "concat/axis" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "concat" op: "ConcatV2" input: "report_uninitialized_variables/boolean_mask/GatherV2" input: "report_uninitialized_resources/Const" input: "concat/axis" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_STRING } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp" op: "VarIsInitializedOp" input: "global_step" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized" op: "IsVariableInitialized" input: "bert/embeddings/word_embeddings" attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_1" op: "IsVariableInitialized" input: "bert/embeddings/token_type_embeddings" attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_2" op: "IsVariableInitialized" input: "bert/embeddings/position_embeddings" attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_3" op: "IsVariableInitialized" input: "bert/embeddings/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_4" op: "IsVariableInitialized" input: "bert/embeddings/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_5" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/self/query/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_6" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/self/query/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_7" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/self/key/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_8" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/self/key/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_9" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/self/value/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_10" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/self/value/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_11" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_12" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_13" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_14" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_15" op: "IsVariableInitialized" input: "bert/encoder/layer_0/intermediate/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_16" op: "IsVariableInitialized" input: "bert/encoder/layer_0/intermediate/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_17" op: "IsVariableInitialized" input: "bert/encoder/layer_0/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_18" op: "IsVariableInitialized" input: "bert/encoder/layer_0/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_19" op: "IsVariableInitialized" input: "bert/encoder/layer_0/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_20" op: "IsVariableInitialized" input: "bert/encoder/layer_0/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_21" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/self/query/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_22" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/self/query/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_23" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/self/key/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_24" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/self/key/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_25" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/self/value/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_26" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/self/value/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_27" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_28" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_29" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_30" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_31" op: "IsVariableInitialized" input: "bert/encoder/layer_1/intermediate/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_32" op: "IsVariableInitialized" input: "bert/encoder/layer_1/intermediate/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_33" op: "IsVariableInitialized" input: "bert/encoder/layer_1/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_34" op: "IsVariableInitialized" input: "bert/encoder/layer_1/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_35" op: "IsVariableInitialized" input: "bert/encoder/layer_1/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_36" op: "IsVariableInitialized" input: "bert/encoder/layer_1/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_37" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/self/query/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_38" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/self/query/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_39" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/self/key/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_40" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/self/key/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_41" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/self/value/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_42" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/self/value/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_43" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_44" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_45" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_46" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_47" op: "IsVariableInitialized" input: "bert/encoder/layer_2/intermediate/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_48" op: "IsVariableInitialized" input: "bert/encoder/layer_2/intermediate/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_49" op: "IsVariableInitialized" input: "bert/encoder/layer_2/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_50" op: "IsVariableInitialized" input: "bert/encoder/layer_2/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_51" op: "IsVariableInitialized" input: "bert/encoder/layer_2/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_52" op: "IsVariableInitialized" input: "bert/encoder/layer_2/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_53" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/self/query/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_54" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/self/query/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_55" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/self/key/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_56" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/self/key/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_57" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/self/value/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_58" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/self/value/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_59" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_60" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_61" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_62" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_63" op: "IsVariableInitialized" input: "bert/encoder/layer_3/intermediate/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_64" op: "IsVariableInitialized" input: "bert/encoder/layer_3/intermediate/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_65" op: "IsVariableInitialized" input: "bert/encoder/layer_3/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_66" op: "IsVariableInitialized" input: "bert/encoder/layer_3/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_67" op: "IsVariableInitialized" input: "bert/encoder/layer_3/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_68" op: "IsVariableInitialized" input: "bert/encoder/layer_3/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_69" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/self/query/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_70" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/self/query/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_71" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/self/key/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_72" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/self/key/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_73" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/self/value/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_74" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/self/value/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_75" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_76" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_77" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_78" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_79" op: "IsVariableInitialized" input: "bert/encoder/layer_4/intermediate/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_80" op: "IsVariableInitialized" input: "bert/encoder/layer_4/intermediate/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_81" op: "IsVariableInitialized" input: "bert/encoder/layer_4/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_82" op: "IsVariableInitialized" input: "bert/encoder/layer_4/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_83" op: "IsVariableInitialized" input: "bert/encoder/layer_4/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_84" op: "IsVariableInitialized" input: "bert/encoder/layer_4/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_85" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/self/query/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_86" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/self/query/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_87" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/self/key/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_88" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/self/key/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_89" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/self/value/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_90" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/self/value/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_91" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_92" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_93" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_94" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_95" op: "IsVariableInitialized" input: "bert/encoder/layer_5/intermediate/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_96" op: "IsVariableInitialized" input: "bert/encoder/layer_5/intermediate/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_97" op: "IsVariableInitialized" input: "bert/encoder/layer_5/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_98" op: "IsVariableInitialized" input: "bert/encoder/layer_5/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_99" op: "IsVariableInitialized" input: "bert/encoder/layer_5/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_100" op: "IsVariableInitialized" input: "bert/encoder/layer_5/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_101" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/self/query/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_102" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/self/query/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_103" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/self/key/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_104" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/self/key/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_105" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/self/value/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_106" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/self/value/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_107" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_108" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_109" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_110" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_111" op: "IsVariableInitialized" input: "bert/encoder/layer_6/intermediate/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_112" op: "IsVariableInitialized" input: "bert/encoder/layer_6/intermediate/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_113" op: "IsVariableInitialized" input: "bert/encoder/layer_6/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_114" op: "IsVariableInitialized" input: "bert/encoder/layer_6/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_115" op: "IsVariableInitialized" input: "bert/encoder/layer_6/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_116" op: "IsVariableInitialized" input: "bert/encoder/layer_6/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_117" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/self/query/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_118" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/self/query/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_119" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/self/key/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_120" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/self/key/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_121" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/self/value/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_122" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/self/value/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_123" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_124" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_125" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_126" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_127" op: "IsVariableInitialized" input: "bert/encoder/layer_7/intermediate/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_128" op: "IsVariableInitialized" input: "bert/encoder/layer_7/intermediate/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_129" op: "IsVariableInitialized" input: "bert/encoder/layer_7/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_130" op: "IsVariableInitialized" input: "bert/encoder/layer_7/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_131" op: "IsVariableInitialized" input: "bert/encoder/layer_7/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_132" op: "IsVariableInitialized" input: "bert/encoder/layer_7/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_133" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/self/query/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_134" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/self/query/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_135" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/self/key/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_136" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/self/key/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_137" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/self/value/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_138" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/self/value/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_139" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_140" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_141" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_142" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_143" op: "IsVariableInitialized" input: "bert/encoder/layer_8/intermediate/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_144" op: "IsVariableInitialized" input: "bert/encoder/layer_8/intermediate/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_145" op: "IsVariableInitialized" input: "bert/encoder/layer_8/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_146" op: "IsVariableInitialized" input: "bert/encoder/layer_8/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_147" op: "IsVariableInitialized" input: "bert/encoder/layer_8/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_148" op: "IsVariableInitialized" input: "bert/encoder/layer_8/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_149" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/self/query/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_150" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/self/query/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_151" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/self/key/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_152" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/self/key/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_153" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/self/value/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_154" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/self/value/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_155" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_156" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_157" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_158" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_159" op: "IsVariableInitialized" input: "bert/encoder/layer_9/intermediate/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_160" op: "IsVariableInitialized" input: "bert/encoder/layer_9/intermediate/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_161" op: "IsVariableInitialized" input: "bert/encoder/layer_9/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_162" op: "IsVariableInitialized" input: "bert/encoder/layer_9/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_163" op: "IsVariableInitialized" input: "bert/encoder/layer_9/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_164" op: "IsVariableInitialized" input: "bert/encoder/layer_9/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_165" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/self/query/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_166" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/self/query/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_167" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/self/key/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_168" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/self/key/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_169" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/self/value/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_170" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/self/value/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_171" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_172" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_173" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_174" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_175" op: "IsVariableInitialized" input: "bert/encoder/layer_10/intermediate/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_176" op: "IsVariableInitialized" input: "bert/encoder/layer_10/intermediate/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_177" op: "IsVariableInitialized" input: "bert/encoder/layer_10/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_178" op: "IsVariableInitialized" input: "bert/encoder/layer_10/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_179" op: "IsVariableInitialized" input: "bert/encoder/layer_10/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_180" op: "IsVariableInitialized" input: "bert/encoder/layer_10/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_181" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/self/query/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_182" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/self/query/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_183" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/self/key/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_184" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/self/key/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_185" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/self/value/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_186" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/self/value/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_187" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_188" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_189" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_190" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_191" op: "IsVariableInitialized" input: "bert/encoder/layer_11/intermediate/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_192" op: "IsVariableInitialized" input: "bert/encoder/layer_11/intermediate/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_193" op: "IsVariableInitialized" input: "bert/encoder/layer_11/output/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_194" op: "IsVariableInitialized" input: "bert/encoder/layer_11/output/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_195" op: "IsVariableInitialized" input: "bert/encoder/layer_11/output/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_196" op: "IsVariableInitialized" input: "bert/encoder/layer_11/output/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_197" op: "IsVariableInitialized" input: "bert/pooler/dense/kernel" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_198" op: "IsVariableInitialized" input: "bert/pooler/dense/bias" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_199" op: "IsVariableInitialized" input: "cls/predictions/transform/dense/kernel" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_200" op: "IsVariableInitialized" input: "cls/predictions/transform/dense/bias" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_201" op: "IsVariableInitialized" input: "cls/predictions/transform/LayerNorm/beta" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_202" op: "IsVariableInitialized" input: "cls/predictions/transform/LayerNorm/gamma" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_203" op: "IsVariableInitialized" input: "cls/predictions/output_bias" attr { key: "_class" value { list { s: "loc:@cls/predictions/output_bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_204" op: "IsVariableInitialized" input: "cls/seq_relationship/output_weights" attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_weights" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_205" op: "IsVariableInitialized" input: "cls/seq_relationship/output_bias" attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_206" op: "IsVariableInitialized" input: "bert/embeddings/word_embeddings/adam_m" attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_207" op: "IsVariableInitialized" input: "bert/embeddings/word_embeddings/adam_v" attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_208" op: "IsVariableInitialized" input: "bert/embeddings/token_type_embeddings/adam_m" attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_209" op: "IsVariableInitialized" input: "bert/embeddings/token_type_embeddings/adam_v" attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_210" op: "IsVariableInitialized" input: "bert/embeddings/position_embeddings/adam_m" attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_211" op: "IsVariableInitialized" input: "bert/embeddings/position_embeddings/adam_v" attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_212" op: "IsVariableInitialized" input: "bert/embeddings/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_213" op: "IsVariableInitialized" input: "bert/embeddings/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_214" op: "IsVariableInitialized" input: "bert/embeddings/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_215" op: "IsVariableInitialized" input: "bert/embeddings/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_216" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_217" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_218" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/self/query/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_219" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/self/query/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_220" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_221" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_222" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/self/key/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_223" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/self/key/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_224" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_225" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_226" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/self/value/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_227" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/self/value/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_228" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_229" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_230" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_231" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_232" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_233" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_234" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_235" op: "IsVariableInitialized" input: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_236" op: "IsVariableInitialized" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_237" op: "IsVariableInitialized" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_238" op: "IsVariableInitialized" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_239" op: "IsVariableInitialized" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_240" op: "IsVariableInitialized" input: "bert/encoder/layer_0/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_241" op: "IsVariableInitialized" input: "bert/encoder/layer_0/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_242" op: "IsVariableInitialized" input: "bert/encoder/layer_0/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_243" op: "IsVariableInitialized" input: "bert/encoder/layer_0/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_244" op: "IsVariableInitialized" input: "bert/encoder/layer_0/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_245" op: "IsVariableInitialized" input: "bert/encoder/layer_0/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_246" op: "IsVariableInitialized" input: "bert/encoder/layer_0/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_247" op: "IsVariableInitialized" input: "bert/encoder/layer_0/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_248" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_249" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_250" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/self/query/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_251" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/self/query/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_252" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_253" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_254" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/self/key/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_255" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/self/key/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_256" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_257" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_258" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/self/value/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_259" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/self/value/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_260" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_261" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_262" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_263" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_264" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_265" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_266" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_267" op: "IsVariableInitialized" input: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_268" op: "IsVariableInitialized" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_269" op: "IsVariableInitialized" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_270" op: "IsVariableInitialized" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_271" op: "IsVariableInitialized" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_272" op: "IsVariableInitialized" input: "bert/encoder/layer_1/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_273" op: "IsVariableInitialized" input: "bert/encoder/layer_1/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_274" op: "IsVariableInitialized" input: "bert/encoder/layer_1/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_275" op: "IsVariableInitialized" input: "bert/encoder/layer_1/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_276" op: "IsVariableInitialized" input: "bert/encoder/layer_1/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_277" op: "IsVariableInitialized" input: "bert/encoder/layer_1/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_278" op: "IsVariableInitialized" input: "bert/encoder/layer_1/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_279" op: "IsVariableInitialized" input: "bert/encoder/layer_1/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_280" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_281" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_282" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/self/query/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_283" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/self/query/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_284" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_285" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_286" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/self/key/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_287" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/self/key/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_288" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_289" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_290" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/self/value/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_291" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/self/value/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_292" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_293" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_294" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_295" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_296" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_297" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_298" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_299" op: "IsVariableInitialized" input: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_300" op: "IsVariableInitialized" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_301" op: "IsVariableInitialized" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_302" op: "IsVariableInitialized" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_303" op: "IsVariableInitialized" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_304" op: "IsVariableInitialized" input: "bert/encoder/layer_2/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_305" op: "IsVariableInitialized" input: "bert/encoder/layer_2/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_306" op: "IsVariableInitialized" input: "bert/encoder/layer_2/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_307" op: "IsVariableInitialized" input: "bert/encoder/layer_2/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_308" op: "IsVariableInitialized" input: "bert/encoder/layer_2/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_309" op: "IsVariableInitialized" input: "bert/encoder/layer_2/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_310" op: "IsVariableInitialized" input: "bert/encoder/layer_2/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_311" op: "IsVariableInitialized" input: "bert/encoder/layer_2/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_312" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_313" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_314" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/self/query/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_315" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/self/query/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_316" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_317" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_318" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/self/key/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_319" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/self/key/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_320" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_321" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_322" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/self/value/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_323" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/self/value/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_324" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_325" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_326" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_327" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_328" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_329" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_330" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_331" op: "IsVariableInitialized" input: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_332" op: "IsVariableInitialized" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_333" op: "IsVariableInitialized" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_334" op: "IsVariableInitialized" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_335" op: "IsVariableInitialized" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_336" op: "IsVariableInitialized" input: "bert/encoder/layer_3/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_337" op: "IsVariableInitialized" input: "bert/encoder/layer_3/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_338" op: "IsVariableInitialized" input: "bert/encoder/layer_3/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_339" op: "IsVariableInitialized" input: "bert/encoder/layer_3/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_340" op: "IsVariableInitialized" input: "bert/encoder/layer_3/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_341" op: "IsVariableInitialized" input: "bert/encoder/layer_3/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_342" op: "IsVariableInitialized" input: "bert/encoder/layer_3/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_343" op: "IsVariableInitialized" input: "bert/encoder/layer_3/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_344" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_345" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_346" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/self/query/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_347" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/self/query/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_348" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_349" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_350" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/self/key/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_351" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/self/key/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_352" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_353" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_354" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/self/value/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_355" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/self/value/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_356" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_357" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_358" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_359" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_360" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_361" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_362" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_363" op: "IsVariableInitialized" input: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_364" op: "IsVariableInitialized" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_365" op: "IsVariableInitialized" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_366" op: "IsVariableInitialized" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_367" op: "IsVariableInitialized" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_368" op: "IsVariableInitialized" input: "bert/encoder/layer_4/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_369" op: "IsVariableInitialized" input: "bert/encoder/layer_4/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_370" op: "IsVariableInitialized" input: "bert/encoder/layer_4/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_371" op: "IsVariableInitialized" input: "bert/encoder/layer_4/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_372" op: "IsVariableInitialized" input: "bert/encoder/layer_4/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_373" op: "IsVariableInitialized" input: "bert/encoder/layer_4/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_374" op: "IsVariableInitialized" input: "bert/encoder/layer_4/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_375" op: "IsVariableInitialized" input: "bert/encoder/layer_4/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_376" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_377" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_378" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/self/query/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_379" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/self/query/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_380" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_381" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_382" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/self/key/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_383" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/self/key/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_384" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_385" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_386" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/self/value/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_387" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/self/value/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_388" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_389" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_390" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_391" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_392" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_393" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_394" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_395" op: "IsVariableInitialized" input: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_396" op: "IsVariableInitialized" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_397" op: "IsVariableInitialized" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_398" op: "IsVariableInitialized" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_399" op: "IsVariableInitialized" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_400" op: "IsVariableInitialized" input: "bert/encoder/layer_5/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_401" op: "IsVariableInitialized" input: "bert/encoder/layer_5/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_402" op: "IsVariableInitialized" input: "bert/encoder/layer_5/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_403" op: "IsVariableInitialized" input: "bert/encoder/layer_5/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_404" op: "IsVariableInitialized" input: "bert/encoder/layer_5/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_405" op: "IsVariableInitialized" input: "bert/encoder/layer_5/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_406" op: "IsVariableInitialized" input: "bert/encoder/layer_5/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_407" op: "IsVariableInitialized" input: "bert/encoder/layer_5/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_408" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_409" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_410" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/self/query/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_411" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/self/query/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_412" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_413" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_414" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/self/key/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_415" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/self/key/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_416" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_417" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_418" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/self/value/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_419" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/self/value/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_420" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_421" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_422" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_423" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_424" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_425" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_426" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_427" op: "IsVariableInitialized" input: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_428" op: "IsVariableInitialized" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_429" op: "IsVariableInitialized" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_430" op: "IsVariableInitialized" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_431" op: "IsVariableInitialized" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_432" op: "IsVariableInitialized" input: "bert/encoder/layer_6/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_433" op: "IsVariableInitialized" input: "bert/encoder/layer_6/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_434" op: "IsVariableInitialized" input: "bert/encoder/layer_6/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_435" op: "IsVariableInitialized" input: "bert/encoder/layer_6/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_436" op: "IsVariableInitialized" input: "bert/encoder/layer_6/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_437" op: "IsVariableInitialized" input: "bert/encoder/layer_6/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_438" op: "IsVariableInitialized" input: "bert/encoder/layer_6/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_439" op: "IsVariableInitialized" input: "bert/encoder/layer_6/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_440" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_441" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_442" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/self/query/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_443" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/self/query/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_444" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_445" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_446" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/self/key/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_447" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/self/key/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_448" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_449" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_450" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/self/value/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_451" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/self/value/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_452" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_453" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_454" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_455" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_456" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_457" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_458" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_459" op: "IsVariableInitialized" input: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_460" op: "IsVariableInitialized" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_461" op: "IsVariableInitialized" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_462" op: "IsVariableInitialized" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_463" op: "IsVariableInitialized" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_464" op: "IsVariableInitialized" input: "bert/encoder/layer_7/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_465" op: "IsVariableInitialized" input: "bert/encoder/layer_7/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_466" op: "IsVariableInitialized" input: "bert/encoder/layer_7/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_467" op: "IsVariableInitialized" input: "bert/encoder/layer_7/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_468" op: "IsVariableInitialized" input: "bert/encoder/layer_7/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_469" op: "IsVariableInitialized" input: "bert/encoder/layer_7/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_470" op: "IsVariableInitialized" input: "bert/encoder/layer_7/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_471" op: "IsVariableInitialized" input: "bert/encoder/layer_7/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_472" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_473" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_474" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/self/query/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_475" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/self/query/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_476" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_477" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_478" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/self/key/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_479" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/self/key/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_480" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_481" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_482" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/self/value/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_483" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/self/value/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_484" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_485" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_486" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_487" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_488" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_489" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_490" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_491" op: "IsVariableInitialized" input: "bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_492" op: "IsVariableInitialized" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_493" op: "IsVariableInitialized" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_494" op: "IsVariableInitialized" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_495" op: "IsVariableInitialized" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_496" op: "IsVariableInitialized" input: "bert/encoder/layer_8/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_497" op: "IsVariableInitialized" input: "bert/encoder/layer_8/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_498" op: "IsVariableInitialized" input: "bert/encoder/layer_8/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_499" op: "IsVariableInitialized" input: "bert/encoder/layer_8/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_500" op: "IsVariableInitialized" input: "bert/encoder/layer_8/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_501" op: "IsVariableInitialized" input: "bert/encoder/layer_8/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_502" op: "IsVariableInitialized" input: "bert/encoder/layer_8/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_503" op: "IsVariableInitialized" input: "bert/encoder/layer_8/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_504" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_505" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_506" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/self/query/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_507" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/self/query/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_508" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_509" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_510" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/self/key/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_511" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/self/key/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_512" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_513" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_514" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/self/value/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_515" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/self/value/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_516" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_517" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_518" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_519" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_520" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_521" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_522" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_523" op: "IsVariableInitialized" input: "bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_524" op: "IsVariableInitialized" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_525" op: "IsVariableInitialized" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_526" op: "IsVariableInitialized" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_527" op: "IsVariableInitialized" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_528" op: "IsVariableInitialized" input: "bert/encoder/layer_9/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_529" op: "IsVariableInitialized" input: "bert/encoder/layer_9/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_530" op: "IsVariableInitialized" input: "bert/encoder/layer_9/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_531" op: "IsVariableInitialized" input: "bert/encoder/layer_9/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_532" op: "IsVariableInitialized" input: "bert/encoder/layer_9/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_533" op: "IsVariableInitialized" input: "bert/encoder/layer_9/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_534" op: "IsVariableInitialized" input: "bert/encoder/layer_9/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_535" op: "IsVariableInitialized" input: "bert/encoder/layer_9/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_536" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_537" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_538" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/self/query/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_539" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/self/query/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_540" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_541" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_542" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/self/key/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_543" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/self/key/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_544" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_545" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_546" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/self/value/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_547" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/self/value/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_548" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_549" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_550" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_551" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_552" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_553" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_554" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_555" op: "IsVariableInitialized" input: "bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_556" op: "IsVariableInitialized" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_557" op: "IsVariableInitialized" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_558" op: "IsVariableInitialized" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_559" op: "IsVariableInitialized" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_560" op: "IsVariableInitialized" input: "bert/encoder/layer_10/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_561" op: "IsVariableInitialized" input: "bert/encoder/layer_10/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_562" op: "IsVariableInitialized" input: "bert/encoder/layer_10/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_563" op: "IsVariableInitialized" input: "bert/encoder/layer_10/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_564" op: "IsVariableInitialized" input: "bert/encoder/layer_10/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_565" op: "IsVariableInitialized" input: "bert/encoder/layer_10/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_566" op: "IsVariableInitialized" input: "bert/encoder/layer_10/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_567" op: "IsVariableInitialized" input: "bert/encoder/layer_10/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_568" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_569" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_570" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/self/query/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_571" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/self/query/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_572" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_573" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_574" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/self/key/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_575" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/self/key/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_576" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_577" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_578" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/self/value/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_579" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/self/value/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_580" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_581" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_582" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_583" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_584" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_585" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_586" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_587" op: "IsVariableInitialized" input: "bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_588" op: "IsVariableInitialized" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_589" op: "IsVariableInitialized" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_590" op: "IsVariableInitialized" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_591" op: "IsVariableInitialized" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_592" op: "IsVariableInitialized" input: "bert/encoder/layer_11/output/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_593" op: "IsVariableInitialized" input: "bert/encoder/layer_11/output/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_594" op: "IsVariableInitialized" input: "bert/encoder/layer_11/output/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_595" op: "IsVariableInitialized" input: "bert/encoder/layer_11/output/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_596" op: "IsVariableInitialized" input: "bert/encoder/layer_11/output/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_597" op: "IsVariableInitialized" input: "bert/encoder/layer_11/output/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_598" op: "IsVariableInitialized" input: "bert/encoder/layer_11/output/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_599" op: "IsVariableInitialized" input: "bert/encoder/layer_11/output/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_600" op: "IsVariableInitialized" input: "bert/pooler/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_601" op: "IsVariableInitialized" input: "bert/pooler/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_602" op: "IsVariableInitialized" input: "bert/pooler/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_603" op: "IsVariableInitialized" input: "bert/pooler/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_604" op: "IsVariableInitialized" input: "cls/predictions/transform/dense/kernel/adam_m" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_605" op: "IsVariableInitialized" input: "cls/predictions/transform/dense/kernel/adam_v" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_606" op: "IsVariableInitialized" input: "cls/predictions/transform/dense/bias/adam_m" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_607" op: "IsVariableInitialized" input: "cls/predictions/transform/dense/bias/adam_v" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_608" op: "IsVariableInitialized" input: "cls/predictions/transform/LayerNorm/beta/adam_m" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_609" op: "IsVariableInitialized" input: "cls/predictions/transform/LayerNorm/beta/adam_v" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_610" op: "IsVariableInitialized" input: "cls/predictions/transform/LayerNorm/gamma/adam_m" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_611" op: "IsVariableInitialized" input: "cls/predictions/transform/LayerNorm/gamma/adam_v" attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_612" op: "IsVariableInitialized" input: "cls/predictions/output_bias/adam_m" attr { key: "_class" value { list { s: "loc:@cls/predictions/output_bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_613" op: "IsVariableInitialized" input: "cls/predictions/output_bias/adam_v" attr { key: "_class" value { list { s: "loc:@cls/predictions/output_bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_614" op: "IsVariableInitialized" input: "cls/seq_relationship/output_weights/adam_m" attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_weights/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_615" op: "IsVariableInitialized" input: "cls/seq_relationship/output_weights/adam_v" attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_weights/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_616" op: "IsVariableInitialized" input: "cls/seq_relationship/output_bias/adam_m" attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/IsVariableInitialized_617" op: "IsVariableInitialized" input: "cls/seq_relationship/output_bias/adam_v" attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "report_uninitialized_variables_1/stack" op: "Pack" input: "report_uninitialized_variables_1/VarIsInitializedOp" input: "report_uninitialized_variables_1/IsVariableInitialized" input: "report_uninitialized_variables_1/IsVariableInitialized_1" input: "report_uninitialized_variables_1/IsVariableInitialized_2" input: "report_uninitialized_variables_1/IsVariableInitialized_3" input: "report_uninitialized_variables_1/IsVariableInitialized_4" input: "report_uninitialized_variables_1/IsVariableInitialized_5" input: "report_uninitialized_variables_1/IsVariableInitialized_6" input: "report_uninitialized_variables_1/IsVariableInitialized_7" input: "report_uninitialized_variables_1/IsVariableInitialized_8" input: "report_uninitialized_variables_1/IsVariableInitialized_9" input: "report_uninitialized_variables_1/IsVariableInitialized_10" input: "report_uninitialized_variables_1/IsVariableInitialized_11" input: "report_uninitialized_variables_1/IsVariableInitialized_12" input: "report_uninitialized_variables_1/IsVariableInitialized_13" input: "report_uninitialized_variables_1/IsVariableInitialized_14" input: "report_uninitialized_variables_1/IsVariableInitialized_15" input: "report_uninitialized_variables_1/IsVariableInitialized_16" input: "report_uninitialized_variables_1/IsVariableInitialized_17" input: "report_uninitialized_variables_1/IsVariableInitialized_18" input: "report_uninitialized_variables_1/IsVariableInitialized_19" input: "report_uninitialized_variables_1/IsVariableInitialized_20" input: "report_uninitialized_variables_1/IsVariableInitialized_21" input: "report_uninitialized_variables_1/IsVariableInitialized_22" input: "report_uninitialized_variables_1/IsVariableInitialized_23" input: "report_uninitialized_variables_1/IsVariableInitialized_24" input: "report_uninitialized_variables_1/IsVariableInitialized_25" input: "report_uninitialized_variables_1/IsVariableInitialized_26" input: "report_uninitialized_variables_1/IsVariableInitialized_27" input: "report_uninitialized_variables_1/IsVariableInitialized_28" input: "report_uninitialized_variables_1/IsVariableInitialized_29" input: "report_uninitialized_variables_1/IsVariableInitialized_30" input: "report_uninitialized_variables_1/IsVariableInitialized_31" input: "report_uninitialized_variables_1/IsVariableInitialized_32" input: "report_uninitialized_variables_1/IsVariableInitialized_33" input: "report_uninitialized_variables_1/IsVariableInitialized_34" input: "report_uninitialized_variables_1/IsVariableInitialized_35" input: "report_uninitialized_variables_1/IsVariableInitialized_36" input: "report_uninitialized_variables_1/IsVariableInitialized_37" input: "report_uninitialized_variables_1/IsVariableInitialized_38" input: "report_uninitialized_variables_1/IsVariableInitialized_39" input: "report_uninitialized_variables_1/IsVariableInitialized_40" input: "report_uninitialized_variables_1/IsVariableInitialized_41" input: "report_uninitialized_variables_1/IsVariableInitialized_42" input: "report_uninitialized_variables_1/IsVariableInitialized_43" input: "report_uninitialized_variables_1/IsVariableInitialized_44" input: "report_uninitialized_variables_1/IsVariableInitialized_45" input: "report_uninitialized_variables_1/IsVariableInitialized_46" input: "report_uninitialized_variables_1/IsVariableInitialized_47" input: "report_uninitialized_variables_1/IsVariableInitialized_48" input: "report_uninitialized_variables_1/IsVariableInitialized_49" input: "report_uninitialized_variables_1/IsVariableInitialized_50" input: "report_uninitialized_variables_1/IsVariableInitialized_51" input: "report_uninitialized_variables_1/IsVariableInitialized_52" input: "report_uninitialized_variables_1/IsVariableInitialized_53" input: "report_uninitialized_variables_1/IsVariableInitialized_54" input: "report_uninitialized_variables_1/IsVariableInitialized_55" input: "report_uninitialized_variables_1/IsVariableInitialized_56" input: "report_uninitialized_variables_1/IsVariableInitialized_57" input: "report_uninitialized_variables_1/IsVariableInitialized_58" input: "report_uninitialized_variables_1/IsVariableInitialized_59" input: "report_uninitialized_variables_1/IsVariableInitialized_60" input: "report_uninitialized_variables_1/IsVariableInitialized_61" input: "report_uninitialized_variables_1/IsVariableInitialized_62" input: "report_uninitialized_variables_1/IsVariableInitialized_63" input: "report_uninitialized_variables_1/IsVariableInitialized_64" input: "report_uninitialized_variables_1/IsVariableInitialized_65" input: "report_uninitialized_variables_1/IsVariableInitialized_66" input: "report_uninitialized_variables_1/IsVariableInitialized_67" input: "report_uninitialized_variables_1/IsVariableInitialized_68" input: "report_uninitialized_variables_1/IsVariableInitialized_69" input: "report_uninitialized_variables_1/IsVariableInitialized_70" input: "report_uninitialized_variables_1/IsVariableInitialized_71" input: "report_uninitialized_variables_1/IsVariableInitialized_72" input: "report_uninitialized_variables_1/IsVariableInitialized_73" input: "report_uninitialized_variables_1/IsVariableInitialized_74" input: "report_uninitialized_variables_1/IsVariableInitialized_75" input: "report_uninitialized_variables_1/IsVariableInitialized_76" input: "report_uninitialized_variables_1/IsVariableInitialized_77" input: "report_uninitialized_variables_1/IsVariableInitialized_78" input: "report_uninitialized_variables_1/IsVariableInitialized_79" input: "report_uninitialized_variables_1/IsVariableInitialized_80" input: "report_uninitialized_variables_1/IsVariableInitialized_81" input: "report_uninitialized_variables_1/IsVariableInitialized_82" input: "report_uninitialized_variables_1/IsVariableInitialized_83" input: "report_uninitialized_variables_1/IsVariableInitialized_84" input: "report_uninitialized_variables_1/IsVariableInitialized_85" input: "report_uninitialized_variables_1/IsVariableInitialized_86" input: "report_uninitialized_variables_1/IsVariableInitialized_87" input: "report_uninitialized_variables_1/IsVariableInitialized_88" input: "report_uninitialized_variables_1/IsVariableInitialized_89" input: "report_uninitialized_variables_1/IsVariableInitialized_90" input: "report_uninitialized_variables_1/IsVariableInitialized_91" input: "report_uninitialized_variables_1/IsVariableInitialized_92" input: "report_uninitialized_variables_1/IsVariableInitialized_93" input: "report_uninitialized_variables_1/IsVariableInitialized_94" input: "report_uninitialized_variables_1/IsVariableInitialized_95" input: "report_uninitialized_variables_1/IsVariableInitialized_96" input: "report_uninitialized_variables_1/IsVariableInitialized_97" input: "report_uninitialized_variables_1/IsVariableInitialized_98" input: "report_uninitialized_variables_1/IsVariableInitialized_99" input: "report_uninitialized_variables_1/IsVariableInitialized_100" input: "report_uninitialized_variables_1/IsVariableInitialized_101" input: "report_uninitialized_variables_1/IsVariableInitialized_102" input: "report_uninitialized_variables_1/IsVariableInitialized_103" input: "report_uninitialized_variables_1/IsVariableInitialized_104" input: "report_uninitialized_variables_1/IsVariableInitialized_105" input: "report_uninitialized_variables_1/IsVariableInitialized_106" input: "report_uninitialized_variables_1/IsVariableInitialized_107" input: "report_uninitialized_variables_1/IsVariableInitialized_108" input: "report_uninitialized_variables_1/IsVariableInitialized_109" input: "report_uninitialized_variables_1/IsVariableInitialized_110" input: "report_uninitialized_variables_1/IsVariableInitialized_111" input: "report_uninitialized_variables_1/IsVariableInitialized_112" input: "report_uninitialized_variables_1/IsVariableInitialized_113" input: "report_uninitialized_variables_1/IsVariableInitialized_114" input: "report_uninitialized_variables_1/IsVariableInitialized_115" input: "report_uninitialized_variables_1/IsVariableInitialized_116" input: "report_uninitialized_variables_1/IsVariableInitialized_117" input: "report_uninitialized_variables_1/IsVariableInitialized_118" input: "report_uninitialized_variables_1/IsVariableInitialized_119" input: "report_uninitialized_variables_1/IsVariableInitialized_120" input: "report_uninitialized_variables_1/IsVariableInitialized_121" input: "report_uninitialized_variables_1/IsVariableInitialized_122" input: "report_uninitialized_variables_1/IsVariableInitialized_123" input: "report_uninitialized_variables_1/IsVariableInitialized_124" input: "report_uninitialized_variables_1/IsVariableInitialized_125" input: "report_uninitialized_variables_1/IsVariableInitialized_126" input: "report_uninitialized_variables_1/IsVariableInitialized_127" input: "report_uninitialized_variables_1/IsVariableInitialized_128" input: "report_uninitialized_variables_1/IsVariableInitialized_129" input: "report_uninitialized_variables_1/IsVariableInitialized_130" input: "report_uninitialized_variables_1/IsVariableInitialized_131" input: "report_uninitialized_variables_1/IsVariableInitialized_132" input: "report_uninitialized_variables_1/IsVariableInitialized_133" input: "report_uninitialized_variables_1/IsVariableInitialized_134" input: "report_uninitialized_variables_1/IsVariableInitialized_135" input: "report_uninitialized_variables_1/IsVariableInitialized_136" input: "report_uninitialized_variables_1/IsVariableInitialized_137" input: "report_uninitialized_variables_1/IsVariableInitialized_138" input: "report_uninitialized_variables_1/IsVariableInitialized_139" input: "report_uninitialized_variables_1/IsVariableInitialized_140" input: "report_uninitialized_variables_1/IsVariableInitialized_141" input: "report_uninitialized_variables_1/IsVariableInitialized_142" input: "report_uninitialized_variables_1/IsVariableInitialized_143" input: "report_uninitialized_variables_1/IsVariableInitialized_144" input: "report_uninitialized_variables_1/IsVariableInitialized_145" input: "report_uninitialized_variables_1/IsVariableInitialized_146" input: "report_uninitialized_variables_1/IsVariableInitialized_147" input: "report_uninitialized_variables_1/IsVariableInitialized_148" input: "report_uninitialized_variables_1/IsVariableInitialized_149" input: "report_uninitialized_variables_1/IsVariableInitialized_150" input: "report_uninitialized_variables_1/IsVariableInitialized_151" input: "report_uninitialized_variables_1/IsVariableInitialized_152" input: "report_uninitialized_variables_1/IsVariableInitialized_153" input: "report_uninitialized_variables_1/IsVariableInitialized_154" input: "report_uninitialized_variables_1/IsVariableInitialized_155" input: "report_uninitialized_variables_1/IsVariableInitialized_156" input: "report_uninitialized_variables_1/IsVariableInitialized_157" input: "report_uninitialized_variables_1/IsVariableInitialized_158" input: "report_uninitialized_variables_1/IsVariableInitialized_159" input: "report_uninitialized_variables_1/IsVariableInitialized_160" input: "report_uninitialized_variables_1/IsVariableInitialized_161" input: "report_uninitialized_variables_1/IsVariableInitialized_162" input: "report_uninitialized_variables_1/IsVariableInitialized_163" input: "report_uninitialized_variables_1/IsVariableInitialized_164" input: "report_uninitialized_variables_1/IsVariableInitialized_165" input: "report_uninitialized_variables_1/IsVariableInitialized_166" input: "report_uninitialized_variables_1/IsVariableInitialized_167" input: "report_uninitialized_variables_1/IsVariableInitialized_168" input: "report_uninitialized_variables_1/IsVariableInitialized_169" input: "report_uninitialized_variables_1/IsVariableInitialized_170" input: "report_uninitialized_variables_1/IsVariableInitialized_171" input: "report_uninitialized_variables_1/IsVariableInitialized_172" input: "report_uninitialized_variables_1/IsVariableInitialized_173" input: "report_uninitialized_variables_1/IsVariableInitialized_174" input: "report_uninitialized_variables_1/IsVariableInitialized_175" input: "report_uninitialized_variables_1/IsVariableInitialized_176" input: "report_uninitialized_variables_1/IsVariableInitialized_177" input: "report_uninitialized_variables_1/IsVariableInitialized_178" input: "report_uninitialized_variables_1/IsVariableInitialized_179" input: "report_uninitialized_variables_1/IsVariableInitialized_180" input: "report_uninitialized_variables_1/IsVariableInitialized_181" input: "report_uninitialized_variables_1/IsVariableInitialized_182" input: "report_uninitialized_variables_1/IsVariableInitialized_183" input: "report_uninitialized_variables_1/IsVariableInitialized_184" input: "report_uninitialized_variables_1/IsVariableInitialized_185" input: "report_uninitialized_variables_1/IsVariableInitialized_186" input: "report_uninitialized_variables_1/IsVariableInitialized_187" input: "report_uninitialized_variables_1/IsVariableInitialized_188" input: "report_uninitialized_variables_1/IsVariableInitialized_189" input: "report_uninitialized_variables_1/IsVariableInitialized_190" input: "report_uninitialized_variables_1/IsVariableInitialized_191" input: "report_uninitialized_variables_1/IsVariableInitialized_192" input: "report_uninitialized_variables_1/IsVariableInitialized_193" input: "report_uninitialized_variables_1/IsVariableInitialized_194" input: "report_uninitialized_variables_1/IsVariableInitialized_195" input: "report_uninitialized_variables_1/IsVariableInitialized_196" input: "report_uninitialized_variables_1/IsVariableInitialized_197" input: "report_uninitialized_variables_1/IsVariableInitialized_198" input: "report_uninitialized_variables_1/IsVariableInitialized_199" input: "report_uninitialized_variables_1/IsVariableInitialized_200" input: "report_uninitialized_variables_1/IsVariableInitialized_201" input: "report_uninitialized_variables_1/IsVariableInitialized_202" input: "report_uninitialized_variables_1/IsVariableInitialized_203" input: "report_uninitialized_variables_1/IsVariableInitialized_204" input: "report_uninitialized_variables_1/IsVariableInitialized_205" input: "report_uninitialized_variables_1/IsVariableInitialized_206" input: "report_uninitialized_variables_1/IsVariableInitialized_207" input: "report_uninitialized_variables_1/IsVariableInitialized_208" input: "report_uninitialized_variables_1/IsVariableInitialized_209" input: "report_uninitialized_variables_1/IsVariableInitialized_210" input: "report_uninitialized_variables_1/IsVariableInitialized_211" input: "report_uninitialized_variables_1/IsVariableInitialized_212" input: "report_uninitialized_variables_1/IsVariableInitialized_213" input: "report_uninitialized_variables_1/IsVariableInitialized_214" input: "report_uninitialized_variables_1/IsVariableInitialized_215" input: "report_uninitialized_variables_1/IsVariableInitialized_216" input: "report_uninitialized_variables_1/IsVariableInitialized_217" input: "report_uninitialized_variables_1/IsVariableInitialized_218" input: "report_uninitialized_variables_1/IsVariableInitialized_219" input: "report_uninitialized_variables_1/IsVariableInitialized_220" input: "report_uninitialized_variables_1/IsVariableInitialized_221" input: "report_uninitialized_variables_1/IsVariableInitialized_222" input: "report_uninitialized_variables_1/IsVariableInitialized_223" input: "report_uninitialized_variables_1/IsVariableInitialized_224" input: "report_uninitialized_variables_1/IsVariableInitialized_225" input: "report_uninitialized_variables_1/IsVariableInitialized_226" input: "report_uninitialized_variables_1/IsVariableInitialized_227" input: "report_uninitialized_variables_1/IsVariableInitialized_228" input: "report_uninitialized_variables_1/IsVariableInitialized_229" input: "report_uninitialized_variables_1/IsVariableInitialized_230" input: "report_uninitialized_variables_1/IsVariableInitialized_231" input: "report_uninitialized_variables_1/IsVariableInitialized_232" input: "report_uninitialized_variables_1/IsVariableInitialized_233" input: "report_uninitialized_variables_1/IsVariableInitialized_234" input: "report_uninitialized_variables_1/IsVariableInitialized_235" input: "report_uninitialized_variables_1/IsVariableInitialized_236" input: "report_uninitialized_variables_1/IsVariableInitialized_237" input: "report_uninitialized_variables_1/IsVariableInitialized_238" input: "report_uninitialized_variables_1/IsVariableInitialized_239" input: "report_uninitialized_variables_1/IsVariableInitialized_240" input: "report_uninitialized_variables_1/IsVariableInitialized_241" input: "report_uninitialized_variables_1/IsVariableInitialized_242" input: "report_uninitialized_variables_1/IsVariableInitialized_243" input: "report_uninitialized_variables_1/IsVariableInitialized_244" input: "report_uninitialized_variables_1/IsVariableInitialized_245" input: "report_uninitialized_variables_1/IsVariableInitialized_246" input: "report_uninitialized_variables_1/IsVariableInitialized_247" input: "report_uninitialized_variables_1/IsVariableInitialized_248" input: "report_uninitialized_variables_1/IsVariableInitialized_249" input: "report_uninitialized_variables_1/IsVariableInitialized_250" input: "report_uninitialized_variables_1/IsVariableInitialized_251" input: "report_uninitialized_variables_1/IsVariableInitialized_252" input: "report_uninitialized_variables_1/IsVariableInitialized_253" input: "report_uninitialized_variables_1/IsVariableInitialized_254" input: "report_uninitialized_variables_1/IsVariableInitialized_255" input: "report_uninitialized_variables_1/IsVariableInitialized_256" input: "report_uninitialized_variables_1/IsVariableInitialized_257" input: "report_uninitialized_variables_1/IsVariableInitialized_258" input: "report_uninitialized_variables_1/IsVariableInitialized_259" input: "report_uninitialized_variables_1/IsVariableInitialized_260" input: "report_uninitialized_variables_1/IsVariableInitialized_261" input: "report_uninitialized_variables_1/IsVariableInitialized_262" input: "report_uninitialized_variables_1/IsVariableInitialized_263" input: "report_uninitialized_variables_1/IsVariableInitialized_264" input: "report_uninitialized_variables_1/IsVariableInitialized_265" input: "report_uninitialized_variables_1/IsVariableInitialized_266" input: "report_uninitialized_variables_1/IsVariableInitialized_267" input: "report_uninitialized_variables_1/IsVariableInitialized_268" input: "report_uninitialized_variables_1/IsVariableInitialized_269" input: "report_uninitialized_variables_1/IsVariableInitialized_270" input: "report_uninitialized_variables_1/IsVariableInitialized_271" input: "report_uninitialized_variables_1/IsVariableInitialized_272" input: "report_uninitialized_variables_1/IsVariableInitialized_273" input: "report_uninitialized_variables_1/IsVariableInitialized_274" input: "report_uninitialized_variables_1/IsVariableInitialized_275" input: "report_uninitialized_variables_1/IsVariableInitialized_276" input: "report_uninitialized_variables_1/IsVariableInitialized_277" input: "report_uninitialized_variables_1/IsVariableInitialized_278" input: "report_uninitialized_variables_1/IsVariableInitialized_279" input: "report_uninitialized_variables_1/IsVariableInitialized_280" input: "report_uninitialized_variables_1/IsVariableInitialized_281" input: "report_uninitialized_variables_1/IsVariableInitialized_282" input: "report_uninitialized_variables_1/IsVariableInitialized_283" input: "report_uninitialized_variables_1/IsVariableInitialized_284" input: "report_uninitialized_variables_1/IsVariableInitialized_285" input: "report_uninitialized_variables_1/IsVariableInitialized_286" input: "report_uninitialized_variables_1/IsVariableInitialized_287" input: "report_uninitialized_variables_1/IsVariableInitialized_288" input: "report_uninitialized_variables_1/IsVariableInitialized_289" input: "report_uninitialized_variables_1/IsVariableInitialized_290" input: "report_uninitialized_variables_1/IsVariableInitialized_291" input: "report_uninitialized_variables_1/IsVariableInitialized_292" input: "report_uninitialized_variables_1/IsVariableInitialized_293" input: "report_uninitialized_variables_1/IsVariableInitialized_294" input: "report_uninitialized_variables_1/IsVariableInitialized_295" input: "report_uninitialized_variables_1/IsVariableInitialized_296" input: "report_uninitialized_variables_1/IsVariableInitialized_297" input: "report_uninitialized_variables_1/IsVariableInitialized_298" input: "report_uninitialized_variables_1/IsVariableInitialized_299" input: "report_uninitialized_variables_1/IsVariableInitialized_300" input: "report_uninitialized_variables_1/IsVariableInitialized_301" input: "report_uninitialized_variables_1/IsVariableInitialized_302" input: "report_uninitialized_variables_1/IsVariableInitialized_303" input: "report_uninitialized_variables_1/IsVariableInitialized_304" input: "report_uninitialized_variables_1/IsVariableInitialized_305" input: "report_uninitialized_variables_1/IsVariableInitialized_306" input: "report_uninitialized_variables_1/IsVariableInitialized_307" input: "report_uninitialized_variables_1/IsVariableInitialized_308" input: "report_uninitialized_variables_1/IsVariableInitialized_309" input: "report_uninitialized_variables_1/IsVariableInitialized_310" input: "report_uninitialized_variables_1/IsVariableInitialized_311" input: "report_uninitialized_variables_1/IsVariableInitialized_312" input: "report_uninitialized_variables_1/IsVariableInitialized_313" input: "report_uninitialized_variables_1/IsVariableInitialized_314" input: "report_uninitialized_variables_1/IsVariableInitialized_315" input: "report_uninitialized_variables_1/IsVariableInitialized_316" input: "report_uninitialized_variables_1/IsVariableInitialized_317" input: "report_uninitialized_variables_1/IsVariableInitialized_318" input: "report_uninitialized_variables_1/IsVariableInitialized_319" input: "report_uninitialized_variables_1/IsVariableInitialized_320" input: "report_uninitialized_variables_1/IsVariableInitialized_321" input: "report_uninitialized_variables_1/IsVariableInitialized_322" input: "report_uninitialized_variables_1/IsVariableInitialized_323" input: "report_uninitialized_variables_1/IsVariableInitialized_324" input: "report_uninitialized_variables_1/IsVariableInitialized_325" input: "report_uninitialized_variables_1/IsVariableInitialized_326" input: "report_uninitialized_variables_1/IsVariableInitialized_327" input: "report_uninitialized_variables_1/IsVariableInitialized_328" input: "report_uninitialized_variables_1/IsVariableInitialized_329" input: "report_uninitialized_variables_1/IsVariableInitialized_330" input: "report_uninitialized_variables_1/IsVariableInitialized_331" input: "report_uninitialized_variables_1/IsVariableInitialized_332" input: "report_uninitialized_variables_1/IsVariableInitialized_333" input: "report_uninitialized_variables_1/IsVariableInitialized_334" input: "report_uninitialized_variables_1/IsVariableInitialized_335" input: "report_uninitialized_variables_1/IsVariableInitialized_336" input: "report_uninitialized_variables_1/IsVariableInitialized_337" input: "report_uninitialized_variables_1/IsVariableInitialized_338" input: "report_uninitialized_variables_1/IsVariableInitialized_339" input: "report_uninitialized_variables_1/IsVariableInitialized_340" input: "report_uninitialized_variables_1/IsVariableInitialized_341" input: "report_uninitialized_variables_1/IsVariableInitialized_342" input: "report_uninitialized_variables_1/IsVariableInitialized_343" input: "report_uninitialized_variables_1/IsVariableInitialized_344" input: "report_uninitialized_variables_1/IsVariableInitialized_345" input: "report_uninitialized_variables_1/IsVariableInitialized_346" input: "report_uninitialized_variables_1/IsVariableInitialized_347" input: "report_uninitialized_variables_1/IsVariableInitialized_348" input: "report_uninitialized_variables_1/IsVariableInitialized_349" input: "report_uninitialized_variables_1/IsVariableInitialized_350" input: "report_uninitialized_variables_1/IsVariableInitialized_351" input: "report_uninitialized_variables_1/IsVariableInitialized_352" input: "report_uninitialized_variables_1/IsVariableInitialized_353" input: "report_uninitialized_variables_1/IsVariableInitialized_354" input: "report_uninitialized_variables_1/IsVariableInitialized_355" input: "report_uninitialized_variables_1/IsVariableInitialized_356" input: "report_uninitialized_variables_1/IsVariableInitialized_357" input: "report_uninitialized_variables_1/IsVariableInitialized_358" input: "report_uninitialized_variables_1/IsVariableInitialized_359" input: "report_uninitialized_variables_1/IsVariableInitialized_360" input: "report_uninitialized_variables_1/IsVariableInitialized_361" input: "report_uninitialized_variables_1/IsVariableInitialized_362" input: "report_uninitialized_variables_1/IsVariableInitialized_363" input: "report_uninitialized_variables_1/IsVariableInitialized_364" input: "report_uninitialized_variables_1/IsVariableInitialized_365" input: "report_uninitialized_variables_1/IsVariableInitialized_366" input: "report_uninitialized_variables_1/IsVariableInitialized_367" input: "report_uninitialized_variables_1/IsVariableInitialized_368" input: "report_uninitialized_variables_1/IsVariableInitialized_369" input: "report_uninitialized_variables_1/IsVariableInitialized_370" input: "report_uninitialized_variables_1/IsVariableInitialized_371" input: "report_uninitialized_variables_1/IsVariableInitialized_372" input: "report_uninitialized_variables_1/IsVariableInitialized_373" input: "report_uninitialized_variables_1/IsVariableInitialized_374" input: "report_uninitialized_variables_1/IsVariableInitialized_375" input: "report_uninitialized_variables_1/IsVariableInitialized_376" input: "report_uninitialized_variables_1/IsVariableInitialized_377" input: "report_uninitialized_variables_1/IsVariableInitialized_378" input: "report_uninitialized_variables_1/IsVariableInitialized_379" input: "report_uninitialized_variables_1/IsVariableInitialized_380" input: "report_uninitialized_variables_1/IsVariableInitialized_381" input: "report_uninitialized_variables_1/IsVariableInitialized_382" input: "report_uninitialized_variables_1/IsVariableInitialized_383" input: "report_uninitialized_variables_1/IsVariableInitialized_384" input: "report_uninitialized_variables_1/IsVariableInitialized_385" input: "report_uninitialized_variables_1/IsVariableInitialized_386" input: "report_uninitialized_variables_1/IsVariableInitialized_387" input: "report_uninitialized_variables_1/IsVariableInitialized_388" input: "report_uninitialized_variables_1/IsVariableInitialized_389" input: "report_uninitialized_variables_1/IsVariableInitialized_390" input: "report_uninitialized_variables_1/IsVariableInitialized_391" input: "report_uninitialized_variables_1/IsVariableInitialized_392" input: "report_uninitialized_variables_1/IsVariableInitialized_393" input: "report_uninitialized_variables_1/IsVariableInitialized_394" input: "report_uninitialized_variables_1/IsVariableInitialized_395" input: "report_uninitialized_variables_1/IsVariableInitialized_396" input: "report_uninitialized_variables_1/IsVariableInitialized_397" input: "report_uninitialized_variables_1/IsVariableInitialized_398" input: "report_uninitialized_variables_1/IsVariableInitialized_399" input: "report_uninitialized_variables_1/IsVariableInitialized_400" input: "report_uninitialized_variables_1/IsVariableInitialized_401" input: "report_uninitialized_variables_1/IsVariableInitialized_402" input: "report_uninitialized_variables_1/IsVariableInitialized_403" input: "report_uninitialized_variables_1/IsVariableInitialized_404" input: "report_uninitialized_variables_1/IsVariableInitialized_405" input: "report_uninitialized_variables_1/IsVariableInitialized_406" input: "report_uninitialized_variables_1/IsVariableInitialized_407" input: "report_uninitialized_variables_1/IsVariableInitialized_408" input: "report_uninitialized_variables_1/IsVariableInitialized_409" input: "report_uninitialized_variables_1/IsVariableInitialized_410" input: "report_uninitialized_variables_1/IsVariableInitialized_411" input: "report_uninitialized_variables_1/IsVariableInitialized_412" input: "report_uninitialized_variables_1/IsVariableInitialized_413" input: "report_uninitialized_variables_1/IsVariableInitialized_414" input: "report_uninitialized_variables_1/IsVariableInitialized_415" input: "report_uninitialized_variables_1/IsVariableInitialized_416" input: "report_uninitialized_variables_1/IsVariableInitialized_417" input: "report_uninitialized_variables_1/IsVariableInitialized_418" input: "report_uninitialized_variables_1/IsVariableInitialized_419" input: "report_uninitialized_variables_1/IsVariableInitialized_420" input: "report_uninitialized_variables_1/IsVariableInitialized_421" input: "report_uninitialized_variables_1/IsVariableInitialized_422" input: "report_uninitialized_variables_1/IsVariableInitialized_423" input: "report_uninitialized_variables_1/IsVariableInitialized_424" input: "report_uninitialized_variables_1/IsVariableInitialized_425" input: "report_uninitialized_variables_1/IsVariableInitialized_426" input: "report_uninitialized_variables_1/IsVariableInitialized_427" input: "report_uninitialized_variables_1/IsVariableInitialized_428" input: "report_uninitialized_variables_1/IsVariableInitialized_429" input: "report_uninitialized_variables_1/IsVariableInitialized_430" input: "report_uninitialized_variables_1/IsVariableInitialized_431" input: "report_uninitialized_variables_1/IsVariableInitialized_432" input: "report_uninitialized_variables_1/IsVariableInitialized_433" input: "report_uninitialized_variables_1/IsVariableInitialized_434" input: "report_uninitialized_variables_1/IsVariableInitialized_435" input: "report_uninitialized_variables_1/IsVariableInitialized_436" input: "report_uninitialized_variables_1/IsVariableInitialized_437" input: "report_uninitialized_variables_1/IsVariableInitialized_438" input: "report_uninitialized_variables_1/IsVariableInitialized_439" input: "report_uninitialized_variables_1/IsVariableInitialized_440" input: "report_uninitialized_variables_1/IsVariableInitialized_441" input: "report_uninitialized_variables_1/IsVariableInitialized_442" input: "report_uninitialized_variables_1/IsVariableInitialized_443" input: "report_uninitialized_variables_1/IsVariableInitialized_444" input: "report_uninitialized_variables_1/IsVariableInitialized_445" input: "report_uninitialized_variables_1/IsVariableInitialized_446" input: "report_uninitialized_variables_1/IsVariableInitialized_447" input: "report_uninitialized_variables_1/IsVariableInitialized_448" input: "report_uninitialized_variables_1/IsVariableInitialized_449" input: "report_uninitialized_variables_1/IsVariableInitialized_450" input: "report_uninitialized_variables_1/IsVariableInitialized_451" input: "report_uninitialized_variables_1/IsVariableInitialized_452" input: "report_uninitialized_variables_1/IsVariableInitialized_453" input: "report_uninitialized_variables_1/IsVariableInitialized_454" input: "report_uninitialized_variables_1/IsVariableInitialized_455" input: "report_uninitialized_variables_1/IsVariableInitialized_456" input: "report_uninitialized_variables_1/IsVariableInitialized_457" input: "report_uninitialized_variables_1/IsVariableInitialized_458" input: "report_uninitialized_variables_1/IsVariableInitialized_459" input: "report_uninitialized_variables_1/IsVariableInitialized_460" input: "report_uninitialized_variables_1/IsVariableInitialized_461" input: "report_uninitialized_variables_1/IsVariableInitialized_462" input: "report_uninitialized_variables_1/IsVariableInitialized_463" input: "report_uninitialized_variables_1/IsVariableInitialized_464" input: "report_uninitialized_variables_1/IsVariableInitialized_465" input: "report_uninitialized_variables_1/IsVariableInitialized_466" input: "report_uninitialized_variables_1/IsVariableInitialized_467" input: "report_uninitialized_variables_1/IsVariableInitialized_468" input: "report_uninitialized_variables_1/IsVariableInitialized_469" input: "report_uninitialized_variables_1/IsVariableInitialized_470" input: "report_uninitialized_variables_1/IsVariableInitialized_471" input: "report_uninitialized_variables_1/IsVariableInitialized_472" input: "report_uninitialized_variables_1/IsVariableInitialized_473" input: "report_uninitialized_variables_1/IsVariableInitialized_474" input: "report_uninitialized_variables_1/IsVariableInitialized_475" input: "report_uninitialized_variables_1/IsVariableInitialized_476" input: "report_uninitialized_variables_1/IsVariableInitialized_477" input: "report_uninitialized_variables_1/IsVariableInitialized_478" input: "report_uninitialized_variables_1/IsVariableInitialized_479" input: "report_uninitialized_variables_1/IsVariableInitialized_480" input: "report_uninitialized_variables_1/IsVariableInitialized_481" input: "report_uninitialized_variables_1/IsVariableInitialized_482" input: "report_uninitialized_variables_1/IsVariableInitialized_483" input: "report_uninitialized_variables_1/IsVariableInitialized_484" input: "report_uninitialized_variables_1/IsVariableInitialized_485" input: "report_uninitialized_variables_1/IsVariableInitialized_486" input: "report_uninitialized_variables_1/IsVariableInitialized_487" input: "report_uninitialized_variables_1/IsVariableInitialized_488" input: "report_uninitialized_variables_1/IsVariableInitialized_489" input: "report_uninitialized_variables_1/IsVariableInitialized_490" input: "report_uninitialized_variables_1/IsVariableInitialized_491" input: "report_uninitialized_variables_1/IsVariableInitialized_492" input: "report_uninitialized_variables_1/IsVariableInitialized_493" input: "report_uninitialized_variables_1/IsVariableInitialized_494" input: "report_uninitialized_variables_1/IsVariableInitialized_495" input: "report_uninitialized_variables_1/IsVariableInitialized_496" input: "report_uninitialized_variables_1/IsVariableInitialized_497" input: "report_uninitialized_variables_1/IsVariableInitialized_498" input: "report_uninitialized_variables_1/IsVariableInitialized_499" input: "report_uninitialized_variables_1/IsVariableInitialized_500" input: "report_uninitialized_variables_1/IsVariableInitialized_501" input: "report_uninitialized_variables_1/IsVariableInitialized_502" input: "report_uninitialized_variables_1/IsVariableInitialized_503" input: "report_uninitialized_variables_1/IsVariableInitialized_504" input: "report_uninitialized_variables_1/IsVariableInitialized_505" input: "report_uninitialized_variables_1/IsVariableInitialized_506" input: "report_uninitialized_variables_1/IsVariableInitialized_507" input: "report_uninitialized_variables_1/IsVariableInitialized_508" input: "report_uninitialized_variables_1/IsVariableInitialized_509" input: "report_uninitialized_variables_1/IsVariableInitialized_510" input: "report_uninitialized_variables_1/IsVariableInitialized_511" input: "report_uninitialized_variables_1/IsVariableInitialized_512" input: "report_uninitialized_variables_1/IsVariableInitialized_513" input: "report_uninitialized_variables_1/IsVariableInitialized_514" input: "report_uninitialized_variables_1/IsVariableInitialized_515" input: "report_uninitialized_variables_1/IsVariableInitialized_516" input: "report_uninitialized_variables_1/IsVariableInitialized_517" input: "report_uninitialized_variables_1/IsVariableInitialized_518" input: "report_uninitialized_variables_1/IsVariableInitialized_519" input: "report_uninitialized_variables_1/IsVariableInitialized_520" input: "report_uninitialized_variables_1/IsVariableInitialized_521" input: "report_uninitialized_variables_1/IsVariableInitialized_522" input: "report_uninitialized_variables_1/IsVariableInitialized_523" input: "report_uninitialized_variables_1/IsVariableInitialized_524" input: "report_uninitialized_variables_1/IsVariableInitialized_525" input: "report_uninitialized_variables_1/IsVariableInitialized_526" input: "report_uninitialized_variables_1/IsVariableInitialized_527" input: "report_uninitialized_variables_1/IsVariableInitialized_528" input: "report_uninitialized_variables_1/IsVariableInitialized_529" input: "report_uninitialized_variables_1/IsVariableInitialized_530" input: "report_uninitialized_variables_1/IsVariableInitialized_531" input: "report_uninitialized_variables_1/IsVariableInitialized_532" input: "report_uninitialized_variables_1/IsVariableInitialized_533" input: "report_uninitialized_variables_1/IsVariableInitialized_534" input: "report_uninitialized_variables_1/IsVariableInitialized_535" input: "report_uninitialized_variables_1/IsVariableInitialized_536" input: "report_uninitialized_variables_1/IsVariableInitialized_537" input: "report_uninitialized_variables_1/IsVariableInitialized_538" input: "report_uninitialized_variables_1/IsVariableInitialized_539" input: "report_uninitialized_variables_1/IsVariableInitialized_540" input: "report_uninitialized_variables_1/IsVariableInitialized_541" input: "report_uninitialized_variables_1/IsVariableInitialized_542" input: "report_uninitialized_variables_1/IsVariableInitialized_543" input: "report_uninitialized_variables_1/IsVariableInitialized_544" input: "report_uninitialized_variables_1/IsVariableInitialized_545" input: "report_uninitialized_variables_1/IsVariableInitialized_546" input: "report_uninitialized_variables_1/IsVariableInitialized_547" input: "report_uninitialized_variables_1/IsVariableInitialized_548" input: "report_uninitialized_variables_1/IsVariableInitialized_549" input: "report_uninitialized_variables_1/IsVariableInitialized_550" input: "report_uninitialized_variables_1/IsVariableInitialized_551" input: "report_uninitialized_variables_1/IsVariableInitialized_552" input: "report_uninitialized_variables_1/IsVariableInitialized_553" input: "report_uninitialized_variables_1/IsVariableInitialized_554" input: "report_uninitialized_variables_1/IsVariableInitialized_555" input: "report_uninitialized_variables_1/IsVariableInitialized_556" input: "report_uninitialized_variables_1/IsVariableInitialized_557" input: "report_uninitialized_variables_1/IsVariableInitialized_558" input: "report_uninitialized_variables_1/IsVariableInitialized_559" input: "report_uninitialized_variables_1/IsVariableInitialized_560" input: "report_uninitialized_variables_1/IsVariableInitialized_561" input: "report_uninitialized_variables_1/IsVariableInitialized_562" input: "report_uninitialized_variables_1/IsVariableInitialized_563" input: "report_uninitialized_variables_1/IsVariableInitialized_564" input: "report_uninitialized_variables_1/IsVariableInitialized_565" input: "report_uninitialized_variables_1/IsVariableInitialized_566" input: "report_uninitialized_variables_1/IsVariableInitialized_567" input: "report_uninitialized_variables_1/IsVariableInitialized_568" input: "report_uninitialized_variables_1/IsVariableInitialized_569" input: "report_uninitialized_variables_1/IsVariableInitialized_570" input: "report_uninitialized_variables_1/IsVariableInitialized_571" input: "report_uninitialized_variables_1/IsVariableInitialized_572" input: "report_uninitialized_variables_1/IsVariableInitialized_573" input: "report_uninitialized_variables_1/IsVariableInitialized_574" input: "report_uninitialized_variables_1/IsVariableInitialized_575" input: "report_uninitialized_variables_1/IsVariableInitialized_576" input: "report_uninitialized_variables_1/IsVariableInitialized_577" input: "report_uninitialized_variables_1/IsVariableInitialized_578" input: "report_uninitialized_variables_1/IsVariableInitialized_579" input: "report_uninitialized_variables_1/IsVariableInitialized_580" input: "report_uninitialized_variables_1/IsVariableInitialized_581" input: "report_uninitialized_variables_1/IsVariableInitialized_582" input: "report_uninitialized_variables_1/IsVariableInitialized_583" input: "report_uninitialized_variables_1/IsVariableInitialized_584" input: "report_uninitialized_variables_1/IsVariableInitialized_585" input: "report_uninitialized_variables_1/IsVariableInitialized_586" input: "report_uninitialized_variables_1/IsVariableInitialized_587" input: "report_uninitialized_variables_1/IsVariableInitialized_588" input: "report_uninitialized_variables_1/IsVariableInitialized_589" input: "report_uninitialized_variables_1/IsVariableInitialized_590" input: "report_uninitialized_variables_1/IsVariableInitialized_591" input: "report_uninitialized_variables_1/IsVariableInitialized_592" input: "report_uninitialized_variables_1/IsVariableInitialized_593" input: "report_uninitialized_variables_1/IsVariableInitialized_594" input: "report_uninitialized_variables_1/IsVariableInitialized_595" input: "report_uninitialized_variables_1/IsVariableInitialized_596" input: "report_uninitialized_variables_1/IsVariableInitialized_597" input: "report_uninitialized_variables_1/IsVariableInitialized_598" input: "report_uninitialized_variables_1/IsVariableInitialized_599" input: "report_uninitialized_variables_1/IsVariableInitialized_600" input: "report_uninitialized_variables_1/IsVariableInitialized_601" input: "report_uninitialized_variables_1/IsVariableInitialized_602" input: "report_uninitialized_variables_1/IsVariableInitialized_603" input: "report_uninitialized_variables_1/IsVariableInitialized_604" input: "report_uninitialized_variables_1/IsVariableInitialized_605" input: "report_uninitialized_variables_1/IsVariableInitialized_606" input: "report_uninitialized_variables_1/IsVariableInitialized_607" input: "report_uninitialized_variables_1/IsVariableInitialized_608" input: "report_uninitialized_variables_1/IsVariableInitialized_609" input: "report_uninitialized_variables_1/IsVariableInitialized_610" input: "report_uninitialized_variables_1/IsVariableInitialized_611" input: "report_uninitialized_variables_1/IsVariableInitialized_612" input: "report_uninitialized_variables_1/IsVariableInitialized_613" input: "report_uninitialized_variables_1/IsVariableInitialized_614" input: "report_uninitialized_variables_1/IsVariableInitialized_615" input: "report_uninitialized_variables_1/IsVariableInitialized_616" input: "report_uninitialized_variables_1/IsVariableInitialized_617" device: "/device:CPU:0" attr { key: "N" value { i: 619 } } attr { key: "T" value { type: DT_BOOL } } attr { key: "_output_shapes" value { list { shape { dim { size: 619 } } } } } attr { key: "axis" value { i: 0 } } } node { name: "report_uninitialized_variables_1/LogicalNot" op: "LogicalNot" input: "report_uninitialized_variables_1/stack" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 619 } } } } } } node { name: "report_uninitialized_variables_1/Const" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 619 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 619 } } string_val: "global_step" string_val: "bert/embeddings/word_embeddings" string_val: "bert/embeddings/token_type_embeddings" string_val: "bert/embeddings/position_embeddings" string_val: "bert/embeddings/LayerNorm/beta" string_val: "bert/embeddings/LayerNorm/gamma" string_val: "bert/encoder/layer_0/attention/self/query/kernel" string_val: "bert/encoder/layer_0/attention/self/query/bias" string_val: "bert/encoder/layer_0/attention/self/key/kernel" string_val: "bert/encoder/layer_0/attention/self/key/bias" string_val: "bert/encoder/layer_0/attention/self/value/kernel" string_val: "bert/encoder/layer_0/attention/self/value/bias" string_val: "bert/encoder/layer_0/attention/output/dense/kernel" string_val: "bert/encoder/layer_0/attention/output/dense/bias" string_val: "bert/encoder/layer_0/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_0/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_0/intermediate/dense/kernel" string_val: "bert/encoder/layer_0/intermediate/dense/bias" string_val: "bert/encoder/layer_0/output/dense/kernel" string_val: "bert/encoder/layer_0/output/dense/bias" string_val: "bert/encoder/layer_0/output/LayerNorm/beta" string_val: "bert/encoder/layer_0/output/LayerNorm/gamma" string_val: "bert/encoder/layer_1/attention/self/query/kernel" string_val: "bert/encoder/layer_1/attention/self/query/bias" string_val: "bert/encoder/layer_1/attention/self/key/kernel" string_val: "bert/encoder/layer_1/attention/self/key/bias" string_val: "bert/encoder/layer_1/attention/self/value/kernel" string_val: "bert/encoder/layer_1/attention/self/value/bias" string_val: "bert/encoder/layer_1/attention/output/dense/kernel" string_val: "bert/encoder/layer_1/attention/output/dense/bias" string_val: "bert/encoder/layer_1/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_1/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_1/intermediate/dense/kernel" string_val: "bert/encoder/layer_1/intermediate/dense/bias" string_val: "bert/encoder/layer_1/output/dense/kernel" string_val: "bert/encoder/layer_1/output/dense/bias" string_val: "bert/encoder/layer_1/output/LayerNorm/beta" string_val: "bert/encoder/layer_1/output/LayerNorm/gamma" string_val: "bert/encoder/layer_2/attention/self/query/kernel" string_val: "bert/encoder/layer_2/attention/self/query/bias" string_val: "bert/encoder/layer_2/attention/self/key/kernel" string_val: "bert/encoder/layer_2/attention/self/key/bias" string_val: "bert/encoder/layer_2/attention/self/value/kernel" string_val: "bert/encoder/layer_2/attention/self/value/bias" string_val: "bert/encoder/layer_2/attention/output/dense/kernel" string_val: "bert/encoder/layer_2/attention/output/dense/bias" string_val: "bert/encoder/layer_2/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_2/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_2/intermediate/dense/kernel" string_val: "bert/encoder/layer_2/intermediate/dense/bias" string_val: "bert/encoder/layer_2/output/dense/kernel" string_val: "bert/encoder/layer_2/output/dense/bias" string_val: "bert/encoder/layer_2/output/LayerNorm/beta" string_val: "bert/encoder/layer_2/output/LayerNorm/gamma" string_val: "bert/encoder/layer_3/attention/self/query/kernel" string_val: "bert/encoder/layer_3/attention/self/query/bias" string_val: "bert/encoder/layer_3/attention/self/key/kernel" string_val: "bert/encoder/layer_3/attention/self/key/bias" string_val: "bert/encoder/layer_3/attention/self/value/kernel" string_val: "bert/encoder/layer_3/attention/self/value/bias" string_val: "bert/encoder/layer_3/attention/output/dense/kernel" string_val: "bert/encoder/layer_3/attention/output/dense/bias" string_val: "bert/encoder/layer_3/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_3/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_3/intermediate/dense/kernel" string_val: "bert/encoder/layer_3/intermediate/dense/bias" string_val: "bert/encoder/layer_3/output/dense/kernel" string_val: "bert/encoder/layer_3/output/dense/bias" string_val: "bert/encoder/layer_3/output/LayerNorm/beta" string_val: "bert/encoder/layer_3/output/LayerNorm/gamma" string_val: "bert/encoder/layer_4/attention/self/query/kernel" string_val: "bert/encoder/layer_4/attention/self/query/bias" string_val: "bert/encoder/layer_4/attention/self/key/kernel" string_val: "bert/encoder/layer_4/attention/self/key/bias" string_val: "bert/encoder/layer_4/attention/self/value/kernel" string_val: "bert/encoder/layer_4/attention/self/value/bias" string_val: "bert/encoder/layer_4/attention/output/dense/kernel" string_val: "bert/encoder/layer_4/attention/output/dense/bias" string_val: "bert/encoder/layer_4/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_4/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_4/intermediate/dense/kernel" string_val: "bert/encoder/layer_4/intermediate/dense/bias" string_val: "bert/encoder/layer_4/output/dense/kernel" string_val: "bert/encoder/layer_4/output/dense/bias" string_val: "bert/encoder/layer_4/output/LayerNorm/beta" string_val: "bert/encoder/layer_4/output/LayerNorm/gamma" string_val: "bert/encoder/layer_5/attention/self/query/kernel" string_val: "bert/encoder/layer_5/attention/self/query/bias" string_val: "bert/encoder/layer_5/attention/self/key/kernel" string_val: "bert/encoder/layer_5/attention/self/key/bias" string_val: "bert/encoder/layer_5/attention/self/value/kernel" string_val: "bert/encoder/layer_5/attention/self/value/bias" string_val: "bert/encoder/layer_5/attention/output/dense/kernel" string_val: "bert/encoder/layer_5/attention/output/dense/bias" string_val: "bert/encoder/layer_5/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_5/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_5/intermediate/dense/kernel" string_val: "bert/encoder/layer_5/intermediate/dense/bias" string_val: "bert/encoder/layer_5/output/dense/kernel" string_val: "bert/encoder/layer_5/output/dense/bias" string_val: "bert/encoder/layer_5/output/LayerNorm/beta" string_val: "bert/encoder/layer_5/output/LayerNorm/gamma" string_val: "bert/encoder/layer_6/attention/self/query/kernel" string_val: "bert/encoder/layer_6/attention/self/query/bias" string_val: "bert/encoder/layer_6/attention/self/key/kernel" string_val: "bert/encoder/layer_6/attention/self/key/bias" string_val: "bert/encoder/layer_6/attention/self/value/kernel" string_val: "bert/encoder/layer_6/attention/self/value/bias" string_val: "bert/encoder/layer_6/attention/output/dense/kernel" string_val: "bert/encoder/layer_6/attention/output/dense/bias" string_val: "bert/encoder/layer_6/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_6/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_6/intermediate/dense/kernel" string_val: "bert/encoder/layer_6/intermediate/dense/bias" string_val: "bert/encoder/layer_6/output/dense/kernel" string_val: "bert/encoder/layer_6/output/dense/bias" string_val: "bert/encoder/layer_6/output/LayerNorm/beta" string_val: "bert/encoder/layer_6/output/LayerNorm/gamma" string_val: "bert/encoder/layer_7/attention/self/query/kernel" string_val: "bert/encoder/layer_7/attention/self/query/bias" string_val: "bert/encoder/layer_7/attention/self/key/kernel" string_val: "bert/encoder/layer_7/attention/self/key/bias" string_val: "bert/encoder/layer_7/attention/self/value/kernel" string_val: "bert/encoder/layer_7/attention/self/value/bias" string_val: "bert/encoder/layer_7/attention/output/dense/kernel" string_val: "bert/encoder/layer_7/attention/output/dense/bias" string_val: "bert/encoder/layer_7/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_7/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_7/intermediate/dense/kernel" string_val: "bert/encoder/layer_7/intermediate/dense/bias" string_val: "bert/encoder/layer_7/output/dense/kernel" string_val: "bert/encoder/layer_7/output/dense/bias" string_val: "bert/encoder/layer_7/output/LayerNorm/beta" string_val: "bert/encoder/layer_7/output/LayerNorm/gamma" string_val: "bert/encoder/layer_8/attention/self/query/kernel" string_val: "bert/encoder/layer_8/attention/self/query/bias" string_val: "bert/encoder/layer_8/attention/self/key/kernel" string_val: "bert/encoder/layer_8/attention/self/key/bias" string_val: "bert/encoder/layer_8/attention/self/value/kernel" string_val: "bert/encoder/layer_8/attention/self/value/bias" string_val: "bert/encoder/layer_8/attention/output/dense/kernel" string_val: "bert/encoder/layer_8/attention/output/dense/bias" string_val: "bert/encoder/layer_8/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_8/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_8/intermediate/dense/kernel" string_val: "bert/encoder/layer_8/intermediate/dense/bias" string_val: "bert/encoder/layer_8/output/dense/kernel" string_val: "bert/encoder/layer_8/output/dense/bias" string_val: "bert/encoder/layer_8/output/LayerNorm/beta" string_val: "bert/encoder/layer_8/output/LayerNorm/gamma" string_val: "bert/encoder/layer_9/attention/self/query/kernel" string_val: "bert/encoder/layer_9/attention/self/query/bias" string_val: "bert/encoder/layer_9/attention/self/key/kernel" string_val: "bert/encoder/layer_9/attention/self/key/bias" string_val: "bert/encoder/layer_9/attention/self/value/kernel" string_val: "bert/encoder/layer_9/attention/self/value/bias" string_val: "bert/encoder/layer_9/attention/output/dense/kernel" string_val: "bert/encoder/layer_9/attention/output/dense/bias" string_val: "bert/encoder/layer_9/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_9/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_9/intermediate/dense/kernel" string_val: "bert/encoder/layer_9/intermediate/dense/bias" string_val: "bert/encoder/layer_9/output/dense/kernel" string_val: "bert/encoder/layer_9/output/dense/bias" string_val: "bert/encoder/layer_9/output/LayerNorm/beta" string_val: "bert/encoder/layer_9/output/LayerNorm/gamma" string_val: "bert/encoder/layer_10/attention/self/query/kernel" string_val: "bert/encoder/layer_10/attention/self/query/bias" string_val: "bert/encoder/layer_10/attention/self/key/kernel" string_val: "bert/encoder/layer_10/attention/self/key/bias" string_val: "bert/encoder/layer_10/attention/self/value/kernel" string_val: "bert/encoder/layer_10/attention/self/value/bias" string_val: "bert/encoder/layer_10/attention/output/dense/kernel" string_val: "bert/encoder/layer_10/attention/output/dense/bias" string_val: "bert/encoder/layer_10/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_10/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_10/intermediate/dense/kernel" string_val: "bert/encoder/layer_10/intermediate/dense/bias" string_val: "bert/encoder/layer_10/output/dense/kernel" string_val: "bert/encoder/layer_10/output/dense/bias" string_val: "bert/encoder/layer_10/output/LayerNorm/beta" string_val: "bert/encoder/layer_10/output/LayerNorm/gamma" string_val: "bert/encoder/layer_11/attention/self/query/kernel" string_val: "bert/encoder/layer_11/attention/self/query/bias" string_val: "bert/encoder/layer_11/attention/self/key/kernel" string_val: "bert/encoder/layer_11/attention/self/key/bias" string_val: "bert/encoder/layer_11/attention/self/value/kernel" string_val: "bert/encoder/layer_11/attention/self/value/bias" string_val: "bert/encoder/layer_11/attention/output/dense/kernel" string_val: "bert/encoder/layer_11/attention/output/dense/bias" string_val: "bert/encoder/layer_11/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_11/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_11/intermediate/dense/kernel" string_val: "bert/encoder/layer_11/intermediate/dense/bias" string_val: "bert/encoder/layer_11/output/dense/kernel" string_val: "bert/encoder/layer_11/output/dense/bias" string_val: "bert/encoder/layer_11/output/LayerNorm/beta" string_val: "bert/encoder/layer_11/output/LayerNorm/gamma" string_val: "bert/pooler/dense/kernel" string_val: "bert/pooler/dense/bias" string_val: "cls/predictions/transform/dense/kernel" string_val: "cls/predictions/transform/dense/bias" string_val: "cls/predictions/transform/LayerNorm/beta" string_val: "cls/predictions/transform/LayerNorm/gamma" string_val: "cls/predictions/output_bias" string_val: "cls/seq_relationship/output_weights" string_val: "cls/seq_relationship/output_bias" string_val: "bert/embeddings/word_embeddings/adam_m" string_val: "bert/embeddings/word_embeddings/adam_v" string_val: "bert/embeddings/token_type_embeddings/adam_m" string_val: "bert/embeddings/token_type_embeddings/adam_v" string_val: "bert/embeddings/position_embeddings/adam_m" string_val: "bert/embeddings/position_embeddings/adam_v" string_val: "bert/embeddings/LayerNorm/beta/adam_m" string_val: "bert/embeddings/LayerNorm/beta/adam_v" string_val: "bert/embeddings/LayerNorm/gamma/adam_m" string_val: "bert/embeddings/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_0/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_0/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_0/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_0/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_0/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_0/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_0/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_0/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_0/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_0/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_0/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_0/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_0/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_0/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_0/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_0/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_0/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_0/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_0/output/dense/bias/adam_m" string_val: "bert/encoder/layer_0/output/dense/bias/adam_v" string_val: "bert/encoder/layer_0/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_0/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_0/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_0/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_1/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_1/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_1/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_1/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_1/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_1/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_1/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_1/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_1/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_1/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_1/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_1/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_1/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_1/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_1/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_1/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_1/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_1/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_1/output/dense/bias/adam_m" string_val: "bert/encoder/layer_1/output/dense/bias/adam_v" string_val: "bert/encoder/layer_1/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_1/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_1/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_1/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_2/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_2/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_2/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_2/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_2/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_2/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_2/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_2/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_2/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_2/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_2/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_2/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_2/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_2/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_2/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_2/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_2/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_2/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_2/output/dense/bias/adam_m" string_val: "bert/encoder/layer_2/output/dense/bias/adam_v" string_val: "bert/encoder/layer_2/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_2/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_2/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_2/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_3/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_3/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_3/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_3/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_3/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_3/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_3/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_3/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_3/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_3/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_3/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_3/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_3/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_3/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_3/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_3/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_3/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_3/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_3/output/dense/bias/adam_m" string_val: "bert/encoder/layer_3/output/dense/bias/adam_v" string_val: "bert/encoder/layer_3/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_3/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_3/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_3/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_4/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_4/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_4/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_4/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_4/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_4/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_4/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_4/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_4/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_4/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_4/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_4/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_4/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_4/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_4/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_4/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_4/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_4/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_4/output/dense/bias/adam_m" string_val: "bert/encoder/layer_4/output/dense/bias/adam_v" string_val: "bert/encoder/layer_4/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_4/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_4/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_4/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_5/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_5/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_5/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_5/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_5/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_5/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_5/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_5/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_5/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_5/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_5/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_5/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_5/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_5/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_5/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_5/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_5/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_5/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_5/output/dense/bias/adam_m" string_val: "bert/encoder/layer_5/output/dense/bias/adam_v" string_val: "bert/encoder/layer_5/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_5/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_5/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_5/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_6/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_6/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_6/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_6/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_6/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_6/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_6/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_6/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_6/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_6/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_6/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_6/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_6/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_6/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_6/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_6/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_6/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_6/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_6/output/dense/bias/adam_m" string_val: "bert/encoder/layer_6/output/dense/bias/adam_v" string_val: "bert/encoder/layer_6/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_6/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_6/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_6/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_7/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_7/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_7/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_7/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_7/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_7/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_7/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_7/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_7/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_7/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_7/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_7/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_7/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_7/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_7/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_7/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_7/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_7/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_7/output/dense/bias/adam_m" string_val: "bert/encoder/layer_7/output/dense/bias/adam_v" string_val: "bert/encoder/layer_7/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_7/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_7/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_7/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_8/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_8/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_8/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_8/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_8/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_8/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_8/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_8/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_8/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_8/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_8/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_8/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_8/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_8/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_8/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_8/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_8/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_8/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_8/output/dense/bias/adam_m" string_val: "bert/encoder/layer_8/output/dense/bias/adam_v" string_val: "bert/encoder/layer_8/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_8/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_8/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_8/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_9/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_9/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_9/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_9/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_9/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_9/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_9/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_9/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_9/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_9/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_9/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_9/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_9/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_9/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_9/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_9/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_9/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_9/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_9/output/dense/bias/adam_m" string_val: "bert/encoder/layer_9/output/dense/bias/adam_v" string_val: "bert/encoder/layer_9/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_9/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_9/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_9/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_10/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_10/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_10/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_10/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_10/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_10/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_10/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_10/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_10/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_10/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_10/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_10/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_10/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_10/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_10/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_10/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_10/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_10/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_10/output/dense/bias/adam_m" string_val: "bert/encoder/layer_10/output/dense/bias/adam_v" string_val: "bert/encoder/layer_10/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_10/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_10/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_10/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_11/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_11/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_11/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_11/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_11/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_11/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_11/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_11/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_11/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_11/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_11/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_11/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_11/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_11/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_11/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_11/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_11/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_11/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_11/output/dense/bias/adam_m" string_val: "bert/encoder/layer_11/output/dense/bias/adam_v" string_val: "bert/encoder/layer_11/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_11/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_11/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_11/output/LayerNorm/gamma/adam_v" string_val: "bert/pooler/dense/kernel/adam_m" string_val: "bert/pooler/dense/kernel/adam_v" string_val: "bert/pooler/dense/bias/adam_m" string_val: "bert/pooler/dense/bias/adam_v" string_val: "cls/predictions/transform/dense/kernel/adam_m" string_val: "cls/predictions/transform/dense/kernel/adam_v" string_val: "cls/predictions/transform/dense/bias/adam_m" string_val: "cls/predictions/transform/dense/bias/adam_v" string_val: "cls/predictions/transform/LayerNorm/beta/adam_m" string_val: "cls/predictions/transform/LayerNorm/beta/adam_v" string_val: "cls/predictions/transform/LayerNorm/gamma/adam_m" string_val: "cls/predictions/transform/LayerNorm/gamma/adam_v" string_val: "cls/predictions/output_bias/adam_m" string_val: "cls/predictions/output_bias/adam_v" string_val: "cls/seq_relationship/output_weights/adam_m" string_val: "cls/seq_relationship/output_weights/adam_v" string_val: "cls/seq_relationship/output_bias/adam_m" string_val: "cls/seq_relationship/output_bias/adam_v" } } } } node { name: "report_uninitialized_variables_1/boolean_mask/Shape" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 619 } } } } node { name: "report_uninitialized_variables_1/boolean_mask/strided_slice/stack" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "report_uninitialized_variables_1/boolean_mask/strided_slice/stack_1" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "report_uninitialized_variables_1/boolean_mask/strided_slice/stack_2" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "report_uninitialized_variables_1/boolean_mask/strided_slice" op: "StridedSlice" input: "report_uninitialized_variables_1/boolean_mask/Shape" input: "report_uninitialized_variables_1/boolean_mask/strided_slice/stack" input: "report_uninitialized_variables_1/boolean_mask/strided_slice/stack_1" input: "report_uninitialized_variables_1/boolean_mask/strided_slice/stack_2" device: "/device:CPU:0" attr { key: "Index" value { type: DT_INT32 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "begin_mask" value { i: 0 } } attr { key: "ellipsis_mask" value { i: 0 } } attr { key: "end_mask" value { i: 0 } } attr { key: "new_axis_mask" value { i: 0 } } attr { key: "shrink_axis_mask" value { i: 0 } } } node { name: "report_uninitialized_variables_1/boolean_mask/Prod/reduction_indices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "report_uninitialized_variables_1/boolean_mask/Prod" op: "Prod" input: "report_uninitialized_variables_1/boolean_mask/strided_slice" input: "report_uninitialized_variables_1/boolean_mask/Prod/reduction_indices" device: "/device:CPU:0" attr { key: "T" value { type: DT_INT32 } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "report_uninitialized_variables_1/boolean_mask/Shape_1" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 619 } } } } node { name: "report_uninitialized_variables_1/boolean_mask/strided_slice_1/stack" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "report_uninitialized_variables_1/boolean_mask/strided_slice_1/stack_1" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "report_uninitialized_variables_1/boolean_mask/strided_slice_1/stack_2" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "report_uninitialized_variables_1/boolean_mask/strided_slice_1" op: "StridedSlice" input: "report_uninitialized_variables_1/boolean_mask/Shape_1" input: "report_uninitialized_variables_1/boolean_mask/strided_slice_1/stack" input: "report_uninitialized_variables_1/boolean_mask/strided_slice_1/stack_1" input: "report_uninitialized_variables_1/boolean_mask/strided_slice_1/stack_2" device: "/device:CPU:0" attr { key: "Index" value { type: DT_INT32 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "begin_mask" value { i: 1 } } attr { key: "ellipsis_mask" value { i: 0 } } attr { key: "end_mask" value { i: 0 } } attr { key: "new_axis_mask" value { i: 0 } } attr { key: "shrink_axis_mask" value { i: 0 } } } node { name: "report_uninitialized_variables_1/boolean_mask/Shape_2" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 619 } } } } node { name: "report_uninitialized_variables_1/boolean_mask/strided_slice_2/stack" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "report_uninitialized_variables_1/boolean_mask/strided_slice_2/stack_1" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "report_uninitialized_variables_1/boolean_mask/strided_slice_2/stack_2" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "report_uninitialized_variables_1/boolean_mask/strided_slice_2" op: "StridedSlice" input: "report_uninitialized_variables_1/boolean_mask/Shape_2" input: "report_uninitialized_variables_1/boolean_mask/strided_slice_2/stack" input: "report_uninitialized_variables_1/boolean_mask/strided_slice_2/stack_1" input: "report_uninitialized_variables_1/boolean_mask/strided_slice_2/stack_2" device: "/device:CPU:0" attr { key: "Index" value { type: DT_INT32 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "begin_mask" value { i: 0 } } attr { key: "ellipsis_mask" value { i: 0 } } attr { key: "end_mask" value { i: 1 } } attr { key: "new_axis_mask" value { i: 0 } } attr { key: "shrink_axis_mask" value { i: 0 } } } node { name: "report_uninitialized_variables_1/boolean_mask/concat/values_1" op: "Pack" input: "report_uninitialized_variables_1/boolean_mask/Prod" device: "/device:CPU:0" attr { key: "N" value { i: 1 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "axis" value { i: 0 } } } node { name: "report_uninitialized_variables_1/boolean_mask/concat/axis" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "report_uninitialized_variables_1/boolean_mask/concat" op: "ConcatV2" input: "report_uninitialized_variables_1/boolean_mask/strided_slice_1" input: "report_uninitialized_variables_1/boolean_mask/concat/values_1" input: "report_uninitialized_variables_1/boolean_mask/strided_slice_2" input: "report_uninitialized_variables_1/boolean_mask/concat/axis" device: "/device:CPU:0" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "report_uninitialized_variables_1/boolean_mask/Reshape" op: "Reshape" input: "report_uninitialized_variables_1/Const" input: "report_uninitialized_variables_1/boolean_mask/concat" device: "/device:CPU:0" attr { key: "T" value { type: DT_STRING } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 619 } } } } } } node { name: "report_uninitialized_variables_1/boolean_mask/Reshape_1/shape" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: -1 } } } } node { name: "report_uninitialized_variables_1/boolean_mask/Reshape_1" op: "Reshape" input: "report_uninitialized_variables_1/LogicalNot" input: "report_uninitialized_variables_1/boolean_mask/Reshape_1/shape" device: "/device:CPU:0" attr { key: "T" value { type: DT_BOOL } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 619 } } } } } } node { name: "report_uninitialized_variables_1/boolean_mask/Where" op: "Where" input: "report_uninitialized_variables_1/boolean_mask/Reshape_1" device: "/device:CPU:0" attr { key: "T" value { type: DT_BOOL } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } dim { size: 1 } } } } } } node { name: "report_uninitialized_variables_1/boolean_mask/Squeeze" op: "Squeeze" input: "report_uninitialized_variables_1/boolean_mask/Where" device: "/device:CPU:0" attr { key: "T" value { type: DT_INT64 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } } } } attr { key: "squeeze_dims" value { list { i: 1 } } } } node { name: "report_uninitialized_variables_1/boolean_mask/GatherV2/axis" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "report_uninitialized_variables_1/boolean_mask/GatherV2" op: "GatherV2" input: "report_uninitialized_variables_1/boolean_mask/Reshape" input: "report_uninitialized_variables_1/boolean_mask/Squeeze" input: "report_uninitialized_variables_1/boolean_mask/GatherV2/axis" device: "/device:CPU:0" attr { key: "Taxis" value { type: DT_INT32 } } attr { key: "Tindices" value { type: DT_INT64 } } attr { key: "Tparams" value { type: DT_STRING } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } } } } } node { name: "init_2" op: "NoOp" } node { name: "init_all_tables" op: "NoOp" } node { name: "init_3" op: "NoOp" } node { name: "group_deps_3" op: "NoOp" input: "^init_2" input: "^init_3" input: "^init_all_tables" } node { name: "Merge/MergeSummary" op: "MergeSummary" input: "loss" attr { key: "N" value { i: 1 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "save/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "model" } } } } node { name: "save/StringJoin/inputs_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "_temp_fbc5fcec492d4393864a35f655d10602/part" } } } } node { name: "save/StringJoin" op: "StringJoin" input: "save/Const" input: "save/StringJoin/inputs_1" attr { key: "N" value { i: 2 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "separator" value { s: "" } } } node { name: "save/num_shards" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "save/ShardedFilename/shard" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "save/ShardedFilename" op: "ShardedFilename" input: "save/StringJoin" input: "save/ShardedFilename/shard" input: "save/num_shards" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "save/SaveV2/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 619 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 619 } } string_val: "bert/embeddings/LayerNorm/beta" string_val: "bert/embeddings/LayerNorm/beta/adam_m" string_val: "bert/embeddings/LayerNorm/beta/adam_v" string_val: "bert/embeddings/LayerNorm/gamma" string_val: "bert/embeddings/LayerNorm/gamma/adam_m" string_val: "bert/embeddings/LayerNorm/gamma/adam_v" string_val: "bert/embeddings/position_embeddings" string_val: "bert/embeddings/position_embeddings/adam_m" string_val: "bert/embeddings/position_embeddings/adam_v" string_val: "bert/embeddings/token_type_embeddings" string_val: "bert/embeddings/token_type_embeddings/adam_m" string_val: "bert/embeddings/token_type_embeddings/adam_v" string_val: "bert/embeddings/word_embeddings" string_val: "bert/embeddings/word_embeddings/adam_m" string_val: "bert/embeddings/word_embeddings/adam_v" string_val: "bert/encoder/layer_0/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_0/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_0/attention/output/dense/bias" string_val: "bert/encoder/layer_0/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_0/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_0/attention/output/dense/kernel" string_val: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_0/attention/self/key/bias" string_val: "bert/encoder/layer_0/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_0/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_0/attention/self/key/kernel" string_val: "bert/encoder/layer_0/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_0/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_0/attention/self/query/bias" string_val: "bert/encoder/layer_0/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_0/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_0/attention/self/query/kernel" string_val: "bert/encoder/layer_0/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_0/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_0/attention/self/value/bias" string_val: "bert/encoder/layer_0/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_0/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_0/attention/self/value/kernel" string_val: "bert/encoder/layer_0/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_0/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_0/intermediate/dense/bias" string_val: "bert/encoder/layer_0/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_0/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_0/intermediate/dense/kernel" string_val: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_0/output/LayerNorm/beta" string_val: "bert/encoder/layer_0/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_0/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_0/output/LayerNorm/gamma" string_val: "bert/encoder/layer_0/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_0/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_0/output/dense/bias" string_val: "bert/encoder/layer_0/output/dense/bias/adam_m" string_val: "bert/encoder/layer_0/output/dense/bias/adam_v" string_val: "bert/encoder/layer_0/output/dense/kernel" string_val: "bert/encoder/layer_0/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_0/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_1/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_1/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_1/attention/output/dense/bias" string_val: "bert/encoder/layer_1/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_1/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_1/attention/output/dense/kernel" string_val: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_1/attention/self/key/bias" string_val: "bert/encoder/layer_1/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_1/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_1/attention/self/key/kernel" string_val: "bert/encoder/layer_1/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_1/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_1/attention/self/query/bias" string_val: "bert/encoder/layer_1/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_1/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_1/attention/self/query/kernel" string_val: "bert/encoder/layer_1/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_1/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_1/attention/self/value/bias" string_val: "bert/encoder/layer_1/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_1/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_1/attention/self/value/kernel" string_val: "bert/encoder/layer_1/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_1/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_1/intermediate/dense/bias" string_val: "bert/encoder/layer_1/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_1/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_1/intermediate/dense/kernel" string_val: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_1/output/LayerNorm/beta" string_val: "bert/encoder/layer_1/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_1/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_1/output/LayerNorm/gamma" string_val: "bert/encoder/layer_1/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_1/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_1/output/dense/bias" string_val: "bert/encoder/layer_1/output/dense/bias/adam_m" string_val: "bert/encoder/layer_1/output/dense/bias/adam_v" string_val: "bert/encoder/layer_1/output/dense/kernel" string_val: "bert/encoder/layer_1/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_1/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_10/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_10/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_10/attention/output/dense/bias" string_val: "bert/encoder/layer_10/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_10/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_10/attention/output/dense/kernel" string_val: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_10/attention/self/key/bias" string_val: "bert/encoder/layer_10/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_10/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_10/attention/self/key/kernel" string_val: "bert/encoder/layer_10/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_10/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_10/attention/self/query/bias" string_val: "bert/encoder/layer_10/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_10/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_10/attention/self/query/kernel" string_val: "bert/encoder/layer_10/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_10/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_10/attention/self/value/bias" string_val: "bert/encoder/layer_10/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_10/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_10/attention/self/value/kernel" string_val: "bert/encoder/layer_10/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_10/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_10/intermediate/dense/bias" string_val: "bert/encoder/layer_10/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_10/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_10/intermediate/dense/kernel" string_val: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_10/output/LayerNorm/beta" string_val: "bert/encoder/layer_10/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_10/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_10/output/LayerNorm/gamma" string_val: "bert/encoder/layer_10/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_10/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_10/output/dense/bias" string_val: "bert/encoder/layer_10/output/dense/bias/adam_m" string_val: "bert/encoder/layer_10/output/dense/bias/adam_v" string_val: "bert/encoder/layer_10/output/dense/kernel" string_val: "bert/encoder/layer_10/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_10/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_11/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_11/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_11/attention/output/dense/bias" string_val: "bert/encoder/layer_11/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_11/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_11/attention/output/dense/kernel" string_val: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_11/attention/self/key/bias" string_val: "bert/encoder/layer_11/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_11/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_11/attention/self/key/kernel" string_val: "bert/encoder/layer_11/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_11/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_11/attention/self/query/bias" string_val: "bert/encoder/layer_11/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_11/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_11/attention/self/query/kernel" string_val: "bert/encoder/layer_11/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_11/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_11/attention/self/value/bias" string_val: "bert/encoder/layer_11/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_11/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_11/attention/self/value/kernel" string_val: "bert/encoder/layer_11/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_11/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_11/intermediate/dense/bias" string_val: "bert/encoder/layer_11/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_11/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_11/intermediate/dense/kernel" string_val: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_11/output/LayerNorm/beta" string_val: "bert/encoder/layer_11/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_11/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_11/output/LayerNorm/gamma" string_val: "bert/encoder/layer_11/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_11/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_11/output/dense/bias" string_val: "bert/encoder/layer_11/output/dense/bias/adam_m" string_val: "bert/encoder/layer_11/output/dense/bias/adam_v" string_val: "bert/encoder/layer_11/output/dense/kernel" string_val: "bert/encoder/layer_11/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_11/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_2/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_2/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_2/attention/output/dense/bias" string_val: "bert/encoder/layer_2/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_2/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_2/attention/output/dense/kernel" string_val: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_2/attention/self/key/bias" string_val: "bert/encoder/layer_2/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_2/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_2/attention/self/key/kernel" string_val: "bert/encoder/layer_2/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_2/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_2/attention/self/query/bias" string_val: "bert/encoder/layer_2/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_2/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_2/attention/self/query/kernel" string_val: "bert/encoder/layer_2/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_2/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_2/attention/self/value/bias" string_val: "bert/encoder/layer_2/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_2/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_2/attention/self/value/kernel" string_val: "bert/encoder/layer_2/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_2/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_2/intermediate/dense/bias" string_val: "bert/encoder/layer_2/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_2/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_2/intermediate/dense/kernel" string_val: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_2/output/LayerNorm/beta" string_val: "bert/encoder/layer_2/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_2/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_2/output/LayerNorm/gamma" string_val: "bert/encoder/layer_2/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_2/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_2/output/dense/bias" string_val: "bert/encoder/layer_2/output/dense/bias/adam_m" string_val: "bert/encoder/layer_2/output/dense/bias/adam_v" string_val: "bert/encoder/layer_2/output/dense/kernel" string_val: "bert/encoder/layer_2/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_2/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_3/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_3/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_3/attention/output/dense/bias" string_val: "bert/encoder/layer_3/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_3/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_3/attention/output/dense/kernel" string_val: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_3/attention/self/key/bias" string_val: "bert/encoder/layer_3/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_3/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_3/attention/self/key/kernel" string_val: "bert/encoder/layer_3/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_3/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_3/attention/self/query/bias" string_val: "bert/encoder/layer_3/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_3/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_3/attention/self/query/kernel" string_val: "bert/encoder/layer_3/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_3/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_3/attention/self/value/bias" string_val: "bert/encoder/layer_3/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_3/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_3/attention/self/value/kernel" string_val: "bert/encoder/layer_3/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_3/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_3/intermediate/dense/bias" string_val: "bert/encoder/layer_3/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_3/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_3/intermediate/dense/kernel" string_val: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_3/output/LayerNorm/beta" string_val: "bert/encoder/layer_3/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_3/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_3/output/LayerNorm/gamma" string_val: "bert/encoder/layer_3/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_3/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_3/output/dense/bias" string_val: "bert/encoder/layer_3/output/dense/bias/adam_m" string_val: "bert/encoder/layer_3/output/dense/bias/adam_v" string_val: "bert/encoder/layer_3/output/dense/kernel" string_val: "bert/encoder/layer_3/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_3/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_4/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_4/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_4/attention/output/dense/bias" string_val: "bert/encoder/layer_4/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_4/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_4/attention/output/dense/kernel" string_val: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_4/attention/self/key/bias" string_val: "bert/encoder/layer_4/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_4/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_4/attention/self/key/kernel" string_val: "bert/encoder/layer_4/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_4/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_4/attention/self/query/bias" string_val: "bert/encoder/layer_4/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_4/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_4/attention/self/query/kernel" string_val: "bert/encoder/layer_4/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_4/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_4/attention/self/value/bias" string_val: "bert/encoder/layer_4/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_4/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_4/attention/self/value/kernel" string_val: "bert/encoder/layer_4/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_4/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_4/intermediate/dense/bias" string_val: "bert/encoder/layer_4/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_4/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_4/intermediate/dense/kernel" string_val: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_4/output/LayerNorm/beta" string_val: "bert/encoder/layer_4/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_4/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_4/output/LayerNorm/gamma" string_val: "bert/encoder/layer_4/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_4/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_4/output/dense/bias" string_val: "bert/encoder/layer_4/output/dense/bias/adam_m" string_val: "bert/encoder/layer_4/output/dense/bias/adam_v" string_val: "bert/encoder/layer_4/output/dense/kernel" string_val: "bert/encoder/layer_4/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_4/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_5/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_5/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_5/attention/output/dense/bias" string_val: "bert/encoder/layer_5/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_5/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_5/attention/output/dense/kernel" string_val: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_5/attention/self/key/bias" string_val: "bert/encoder/layer_5/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_5/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_5/attention/self/key/kernel" string_val: "bert/encoder/layer_5/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_5/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_5/attention/self/query/bias" string_val: "bert/encoder/layer_5/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_5/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_5/attention/self/query/kernel" string_val: "bert/encoder/layer_5/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_5/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_5/attention/self/value/bias" string_val: "bert/encoder/layer_5/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_5/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_5/attention/self/value/kernel" string_val: "bert/encoder/layer_5/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_5/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_5/intermediate/dense/bias" string_val: "bert/encoder/layer_5/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_5/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_5/intermediate/dense/kernel" string_val: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_5/output/LayerNorm/beta" string_val: "bert/encoder/layer_5/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_5/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_5/output/LayerNorm/gamma" string_val: "bert/encoder/layer_5/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_5/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_5/output/dense/bias" string_val: "bert/encoder/layer_5/output/dense/bias/adam_m" string_val: "bert/encoder/layer_5/output/dense/bias/adam_v" string_val: "bert/encoder/layer_5/output/dense/kernel" string_val: "bert/encoder/layer_5/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_5/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_6/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_6/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_6/attention/output/dense/bias" string_val: "bert/encoder/layer_6/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_6/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_6/attention/output/dense/kernel" string_val: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_6/attention/self/key/bias" string_val: "bert/encoder/layer_6/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_6/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_6/attention/self/key/kernel" string_val: "bert/encoder/layer_6/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_6/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_6/attention/self/query/bias" string_val: "bert/encoder/layer_6/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_6/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_6/attention/self/query/kernel" string_val: "bert/encoder/layer_6/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_6/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_6/attention/self/value/bias" string_val: "bert/encoder/layer_6/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_6/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_6/attention/self/value/kernel" string_val: "bert/encoder/layer_6/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_6/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_6/intermediate/dense/bias" string_val: "bert/encoder/layer_6/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_6/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_6/intermediate/dense/kernel" string_val: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_6/output/LayerNorm/beta" string_val: "bert/encoder/layer_6/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_6/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_6/output/LayerNorm/gamma" string_val: "bert/encoder/layer_6/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_6/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_6/output/dense/bias" string_val: "bert/encoder/layer_6/output/dense/bias/adam_m" string_val: "bert/encoder/layer_6/output/dense/bias/adam_v" string_val: "bert/encoder/layer_6/output/dense/kernel" string_val: "bert/encoder/layer_6/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_6/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_7/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_7/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_7/attention/output/dense/bias" string_val: "bert/encoder/layer_7/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_7/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_7/attention/output/dense/kernel" string_val: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_7/attention/self/key/bias" string_val: "bert/encoder/layer_7/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_7/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_7/attention/self/key/kernel" string_val: "bert/encoder/layer_7/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_7/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_7/attention/self/query/bias" string_val: "bert/encoder/layer_7/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_7/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_7/attention/self/query/kernel" string_val: "bert/encoder/layer_7/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_7/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_7/attention/self/value/bias" string_val: "bert/encoder/layer_7/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_7/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_7/attention/self/value/kernel" string_val: "bert/encoder/layer_7/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_7/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_7/intermediate/dense/bias" string_val: "bert/encoder/layer_7/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_7/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_7/intermediate/dense/kernel" string_val: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_7/output/LayerNorm/beta" string_val: "bert/encoder/layer_7/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_7/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_7/output/LayerNorm/gamma" string_val: "bert/encoder/layer_7/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_7/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_7/output/dense/bias" string_val: "bert/encoder/layer_7/output/dense/bias/adam_m" string_val: "bert/encoder/layer_7/output/dense/bias/adam_v" string_val: "bert/encoder/layer_7/output/dense/kernel" string_val: "bert/encoder/layer_7/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_7/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_8/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_8/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_8/attention/output/dense/bias" string_val: "bert/encoder/layer_8/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_8/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_8/attention/output/dense/kernel" string_val: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_8/attention/self/key/bias" string_val: "bert/encoder/layer_8/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_8/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_8/attention/self/key/kernel" string_val: "bert/encoder/layer_8/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_8/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_8/attention/self/query/bias" string_val: "bert/encoder/layer_8/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_8/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_8/attention/self/query/kernel" string_val: "bert/encoder/layer_8/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_8/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_8/attention/self/value/bias" string_val: "bert/encoder/layer_8/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_8/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_8/attention/self/value/kernel" string_val: "bert/encoder/layer_8/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_8/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_8/intermediate/dense/bias" string_val: "bert/encoder/layer_8/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_8/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_8/intermediate/dense/kernel" string_val: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_8/output/LayerNorm/beta" string_val: "bert/encoder/layer_8/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_8/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_8/output/LayerNorm/gamma" string_val: "bert/encoder/layer_8/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_8/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_8/output/dense/bias" string_val: "bert/encoder/layer_8/output/dense/bias/adam_m" string_val: "bert/encoder/layer_8/output/dense/bias/adam_v" string_val: "bert/encoder/layer_8/output/dense/kernel" string_val: "bert/encoder/layer_8/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_8/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_9/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_9/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_9/attention/output/dense/bias" string_val: "bert/encoder/layer_9/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_9/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_9/attention/output/dense/kernel" string_val: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_9/attention/self/key/bias" string_val: "bert/encoder/layer_9/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_9/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_9/attention/self/key/kernel" string_val: "bert/encoder/layer_9/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_9/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_9/attention/self/query/bias" string_val: "bert/encoder/layer_9/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_9/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_9/attention/self/query/kernel" string_val: "bert/encoder/layer_9/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_9/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_9/attention/self/value/bias" string_val: "bert/encoder/layer_9/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_9/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_9/attention/self/value/kernel" string_val: "bert/encoder/layer_9/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_9/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_9/intermediate/dense/bias" string_val: "bert/encoder/layer_9/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_9/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_9/intermediate/dense/kernel" string_val: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_9/output/LayerNorm/beta" string_val: "bert/encoder/layer_9/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_9/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_9/output/LayerNorm/gamma" string_val: "bert/encoder/layer_9/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_9/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_9/output/dense/bias" string_val: "bert/encoder/layer_9/output/dense/bias/adam_m" string_val: "bert/encoder/layer_9/output/dense/bias/adam_v" string_val: "bert/encoder/layer_9/output/dense/kernel" string_val: "bert/encoder/layer_9/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_9/output/dense/kernel/adam_v" string_val: "bert/pooler/dense/bias" string_val: "bert/pooler/dense/bias/adam_m" string_val: "bert/pooler/dense/bias/adam_v" string_val: "bert/pooler/dense/kernel" string_val: "bert/pooler/dense/kernel/adam_m" string_val: "bert/pooler/dense/kernel/adam_v" string_val: "cls/predictions/output_bias" string_val: "cls/predictions/output_bias/adam_m" string_val: "cls/predictions/output_bias/adam_v" string_val: "cls/predictions/transform/LayerNorm/beta" string_val: "cls/predictions/transform/LayerNorm/beta/adam_m" string_val: "cls/predictions/transform/LayerNorm/beta/adam_v" string_val: "cls/predictions/transform/LayerNorm/gamma" string_val: "cls/predictions/transform/LayerNorm/gamma/adam_m" string_val: "cls/predictions/transform/LayerNorm/gamma/adam_v" string_val: "cls/predictions/transform/dense/bias" string_val: "cls/predictions/transform/dense/bias/adam_m" string_val: "cls/predictions/transform/dense/bias/adam_v" string_val: "cls/predictions/transform/dense/kernel" string_val: "cls/predictions/transform/dense/kernel/adam_m" string_val: "cls/predictions/transform/dense/kernel/adam_v" string_val: "cls/seq_relationship/output_bias" string_val: "cls/seq_relationship/output_bias/adam_m" string_val: "cls/seq_relationship/output_bias/adam_v" string_val: "cls/seq_relationship/output_weights" string_val: "cls/seq_relationship/output_weights/adam_m" string_val: "cls/seq_relationship/output_weights/adam_v" string_val: "global_step" } } } } node { name: "save/SaveV2/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 619 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 619 } } string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" } } } } node { name: "save/SaveV2" op: "SaveV2" input: "save/ShardedFilename" input: "save/SaveV2/tensor_names" input: "save/SaveV2/shape_and_slices" input: "bert/embeddings/LayerNorm/beta" input: "bert/embeddings/LayerNorm/beta/adam_m" input: "bert/embeddings/LayerNorm/beta/adam_v" input: "bert/embeddings/LayerNorm/gamma" input: "bert/embeddings/LayerNorm/gamma/adam_m" input: "bert/embeddings/LayerNorm/gamma/adam_v" input: "bert/embeddings/position_embeddings" input: "bert/embeddings/position_embeddings/adam_m" input: "bert/embeddings/position_embeddings/adam_v" input: "bert/embeddings/token_type_embeddings" input: "bert/embeddings/token_type_embeddings/adam_m" input: "bert/embeddings/token_type_embeddings/adam_v" input: "bert/embeddings/word_embeddings" input: "bert/embeddings/word_embeddings/adam_m" input: "bert/embeddings/word_embeddings/adam_v" input: "bert/encoder/layer_0/attention/output/LayerNorm/beta" input: "bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_0/attention/output/LayerNorm/gamma" input: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_0/attention/output/dense/bias" input: "bert/encoder/layer_0/attention/output/dense/bias/adam_m" input: "bert/encoder/layer_0/attention/output/dense/bias/adam_v" input: "bert/encoder/layer_0/attention/output/dense/kernel" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v" input: "bert/encoder/layer_0/attention/self/key/bias" input: "bert/encoder/layer_0/attention/self/key/bias/adam_m" input: "bert/encoder/layer_0/attention/self/key/bias/adam_v" input: "bert/encoder/layer_0/attention/self/key/kernel" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_m" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_v" input: "bert/encoder/layer_0/attention/self/query/bias" input: "bert/encoder/layer_0/attention/self/query/bias/adam_m" input: "bert/encoder/layer_0/attention/self/query/bias/adam_v" input: "bert/encoder/layer_0/attention/self/query/kernel" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_m" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_v" input: "bert/encoder/layer_0/attention/self/value/bias" input: "bert/encoder/layer_0/attention/self/value/bias/adam_m" input: "bert/encoder/layer_0/attention/self/value/bias/adam_v" input: "bert/encoder/layer_0/attention/self/value/kernel" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_m" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_v" input: "bert/encoder/layer_0/intermediate/dense/bias" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_m" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_v" input: "bert/encoder/layer_0/intermediate/dense/kernel" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v" input: "bert/encoder/layer_0/output/LayerNorm/beta" input: "bert/encoder/layer_0/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_0/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_0/output/LayerNorm/gamma" input: "bert/encoder/layer_0/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_0/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_0/output/dense/bias" input: "bert/encoder/layer_0/output/dense/bias/adam_m" input: "bert/encoder/layer_0/output/dense/bias/adam_v" input: "bert/encoder/layer_0/output/dense/kernel" input: "bert/encoder/layer_0/output/dense/kernel/adam_m" input: "bert/encoder/layer_0/output/dense/kernel/adam_v" input: "bert/encoder/layer_1/attention/output/LayerNorm/beta" input: "bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_1/attention/output/LayerNorm/gamma" input: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_1/attention/output/dense/bias" input: "bert/encoder/layer_1/attention/output/dense/bias/adam_m" input: "bert/encoder/layer_1/attention/output/dense/bias/adam_v" input: "bert/encoder/layer_1/attention/output/dense/kernel" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v" input: "bert/encoder/layer_1/attention/self/key/bias" input: "bert/encoder/layer_1/attention/self/key/bias/adam_m" input: "bert/encoder/layer_1/attention/self/key/bias/adam_v" input: "bert/encoder/layer_1/attention/self/key/kernel" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_m" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_v" input: "bert/encoder/layer_1/attention/self/query/bias" input: "bert/encoder/layer_1/attention/self/query/bias/adam_m" input: "bert/encoder/layer_1/attention/self/query/bias/adam_v" input: "bert/encoder/layer_1/attention/self/query/kernel" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_m" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_v" input: "bert/encoder/layer_1/attention/self/value/bias" input: "bert/encoder/layer_1/attention/self/value/bias/adam_m" input: "bert/encoder/layer_1/attention/self/value/bias/adam_v" input: "bert/encoder/layer_1/attention/self/value/kernel" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_m" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_v" input: "bert/encoder/layer_1/intermediate/dense/bias" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_m" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_v" input: "bert/encoder/layer_1/intermediate/dense/kernel" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v" input: "bert/encoder/layer_1/output/LayerNorm/beta" input: "bert/encoder/layer_1/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_1/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_1/output/LayerNorm/gamma" input: "bert/encoder/layer_1/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_1/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_1/output/dense/bias" input: "bert/encoder/layer_1/output/dense/bias/adam_m" input: "bert/encoder/layer_1/output/dense/bias/adam_v" input: "bert/encoder/layer_1/output/dense/kernel" input: "bert/encoder/layer_1/output/dense/kernel/adam_m" input: "bert/encoder/layer_1/output/dense/kernel/adam_v" input: "bert/encoder/layer_10/attention/output/LayerNorm/beta" input: "bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_10/attention/output/LayerNorm/gamma" input: "bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_10/attention/output/dense/bias" input: "bert/encoder/layer_10/attention/output/dense/bias/adam_m" input: "bert/encoder/layer_10/attention/output/dense/bias/adam_v" input: "bert/encoder/layer_10/attention/output/dense/kernel" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v" input: "bert/encoder/layer_10/attention/self/key/bias" input: "bert/encoder/layer_10/attention/self/key/bias/adam_m" input: "bert/encoder/layer_10/attention/self/key/bias/adam_v" input: "bert/encoder/layer_10/attention/self/key/kernel" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_m" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_v" input: "bert/encoder/layer_10/attention/self/query/bias" input: "bert/encoder/layer_10/attention/self/query/bias/adam_m" input: "bert/encoder/layer_10/attention/self/query/bias/adam_v" input: "bert/encoder/layer_10/attention/self/query/kernel" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_m" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_v" input: "bert/encoder/layer_10/attention/self/value/bias" input: "bert/encoder/layer_10/attention/self/value/bias/adam_m" input: "bert/encoder/layer_10/attention/self/value/bias/adam_v" input: "bert/encoder/layer_10/attention/self/value/kernel" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_m" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_v" input: "bert/encoder/layer_10/intermediate/dense/bias" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_m" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_v" input: "bert/encoder/layer_10/intermediate/dense/kernel" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v" input: "bert/encoder/layer_10/output/LayerNorm/beta" input: "bert/encoder/layer_10/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_10/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_10/output/LayerNorm/gamma" input: "bert/encoder/layer_10/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_10/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_10/output/dense/bias" input: "bert/encoder/layer_10/output/dense/bias/adam_m" input: "bert/encoder/layer_10/output/dense/bias/adam_v" input: "bert/encoder/layer_10/output/dense/kernel" input: "bert/encoder/layer_10/output/dense/kernel/adam_m" input: "bert/encoder/layer_10/output/dense/kernel/adam_v" input: "bert/encoder/layer_11/attention/output/LayerNorm/beta" input: "bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_11/attention/output/LayerNorm/gamma" input: "bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_11/attention/output/dense/bias" input: "bert/encoder/layer_11/attention/output/dense/bias/adam_m" input: "bert/encoder/layer_11/attention/output/dense/bias/adam_v" input: "bert/encoder/layer_11/attention/output/dense/kernel" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v" input: "bert/encoder/layer_11/attention/self/key/bias" input: "bert/encoder/layer_11/attention/self/key/bias/adam_m" input: "bert/encoder/layer_11/attention/self/key/bias/adam_v" input: "bert/encoder/layer_11/attention/self/key/kernel" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_m" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_v" input: "bert/encoder/layer_11/attention/self/query/bias" input: "bert/encoder/layer_11/attention/self/query/bias/adam_m" input: "bert/encoder/layer_11/attention/self/query/bias/adam_v" input: "bert/encoder/layer_11/attention/self/query/kernel" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_m" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_v" input: "bert/encoder/layer_11/attention/self/value/bias" input: "bert/encoder/layer_11/attention/self/value/bias/adam_m" input: "bert/encoder/layer_11/attention/self/value/bias/adam_v" input: "bert/encoder/layer_11/attention/self/value/kernel" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_m" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_v" input: "bert/encoder/layer_11/intermediate/dense/bias" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_m" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_v" input: "bert/encoder/layer_11/intermediate/dense/kernel" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v" input: "bert/encoder/layer_11/output/LayerNorm/beta" input: "bert/encoder/layer_11/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_11/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_11/output/LayerNorm/gamma" input: "bert/encoder/layer_11/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_11/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_11/output/dense/bias" input: "bert/encoder/layer_11/output/dense/bias/adam_m" input: "bert/encoder/layer_11/output/dense/bias/adam_v" input: "bert/encoder/layer_11/output/dense/kernel" input: "bert/encoder/layer_11/output/dense/kernel/adam_m" input: "bert/encoder/layer_11/output/dense/kernel/adam_v" input: "bert/encoder/layer_2/attention/output/LayerNorm/beta" input: "bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_2/attention/output/LayerNorm/gamma" input: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_2/attention/output/dense/bias" input: "bert/encoder/layer_2/attention/output/dense/bias/adam_m" input: "bert/encoder/layer_2/attention/output/dense/bias/adam_v" input: "bert/encoder/layer_2/attention/output/dense/kernel" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v" input: "bert/encoder/layer_2/attention/self/key/bias" input: "bert/encoder/layer_2/attention/self/key/bias/adam_m" input: "bert/encoder/layer_2/attention/self/key/bias/adam_v" input: "bert/encoder/layer_2/attention/self/key/kernel" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_m" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_v" input: "bert/encoder/layer_2/attention/self/query/bias" input: "bert/encoder/layer_2/attention/self/query/bias/adam_m" input: "bert/encoder/layer_2/attention/self/query/bias/adam_v" input: "bert/encoder/layer_2/attention/self/query/kernel" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_m" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_v" input: "bert/encoder/layer_2/attention/self/value/bias" input: "bert/encoder/layer_2/attention/self/value/bias/adam_m" input: "bert/encoder/layer_2/attention/self/value/bias/adam_v" input: "bert/encoder/layer_2/attention/self/value/kernel" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_m" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_v" input: "bert/encoder/layer_2/intermediate/dense/bias" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_m" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_v" input: "bert/encoder/layer_2/intermediate/dense/kernel" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v" input: "bert/encoder/layer_2/output/LayerNorm/beta" input: "bert/encoder/layer_2/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_2/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_2/output/LayerNorm/gamma" input: "bert/encoder/layer_2/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_2/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_2/output/dense/bias" input: "bert/encoder/layer_2/output/dense/bias/adam_m" input: "bert/encoder/layer_2/output/dense/bias/adam_v" input: "bert/encoder/layer_2/output/dense/kernel" input: "bert/encoder/layer_2/output/dense/kernel/adam_m" input: "bert/encoder/layer_2/output/dense/kernel/adam_v" input: "bert/encoder/layer_3/attention/output/LayerNorm/beta" input: "bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_3/attention/output/LayerNorm/gamma" input: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_3/attention/output/dense/bias" input: "bert/encoder/layer_3/attention/output/dense/bias/adam_m" input: "bert/encoder/layer_3/attention/output/dense/bias/adam_v" input: "bert/encoder/layer_3/attention/output/dense/kernel" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v" input: "bert/encoder/layer_3/attention/self/key/bias" input: "bert/encoder/layer_3/attention/self/key/bias/adam_m" input: "bert/encoder/layer_3/attention/self/key/bias/adam_v" input: "bert/encoder/layer_3/attention/self/key/kernel" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_m" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_v" input: "bert/encoder/layer_3/attention/self/query/bias" input: "bert/encoder/layer_3/attention/self/query/bias/adam_m" input: "bert/encoder/layer_3/attention/self/query/bias/adam_v" input: "bert/encoder/layer_3/attention/self/query/kernel" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_m" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_v" input: "bert/encoder/layer_3/attention/self/value/bias" input: "bert/encoder/layer_3/attention/self/value/bias/adam_m" input: "bert/encoder/layer_3/attention/self/value/bias/adam_v" input: "bert/encoder/layer_3/attention/self/value/kernel" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_m" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_v" input: "bert/encoder/layer_3/intermediate/dense/bias" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_m" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_v" input: "bert/encoder/layer_3/intermediate/dense/kernel" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v" input: "bert/encoder/layer_3/output/LayerNorm/beta" input: "bert/encoder/layer_3/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_3/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_3/output/LayerNorm/gamma" input: "bert/encoder/layer_3/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_3/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_3/output/dense/bias" input: "bert/encoder/layer_3/output/dense/bias/adam_m" input: "bert/encoder/layer_3/output/dense/bias/adam_v" input: "bert/encoder/layer_3/output/dense/kernel" input: "bert/encoder/layer_3/output/dense/kernel/adam_m" input: "bert/encoder/layer_3/output/dense/kernel/adam_v" input: "bert/encoder/layer_4/attention/output/LayerNorm/beta" input: "bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_4/attention/output/LayerNorm/gamma" input: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_4/attention/output/dense/bias" input: "bert/encoder/layer_4/attention/output/dense/bias/adam_m" input: "bert/encoder/layer_4/attention/output/dense/bias/adam_v" input: "bert/encoder/layer_4/attention/output/dense/kernel" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v" input: "bert/encoder/layer_4/attention/self/key/bias" input: "bert/encoder/layer_4/attention/self/key/bias/adam_m" input: "bert/encoder/layer_4/attention/self/key/bias/adam_v" input: "bert/encoder/layer_4/attention/self/key/kernel" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_m" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_v" input: "bert/encoder/layer_4/attention/self/query/bias" input: "bert/encoder/layer_4/attention/self/query/bias/adam_m" input: "bert/encoder/layer_4/attention/self/query/bias/adam_v" input: "bert/encoder/layer_4/attention/self/query/kernel" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_m" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_v" input: "bert/encoder/layer_4/attention/self/value/bias" input: "bert/encoder/layer_4/attention/self/value/bias/adam_m" input: "bert/encoder/layer_4/attention/self/value/bias/adam_v" input: "bert/encoder/layer_4/attention/self/value/kernel" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_m" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_v" input: "bert/encoder/layer_4/intermediate/dense/bias" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_m" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_v" input: "bert/encoder/layer_4/intermediate/dense/kernel" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v" input: "bert/encoder/layer_4/output/LayerNorm/beta" input: "bert/encoder/layer_4/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_4/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_4/output/LayerNorm/gamma" input: "bert/encoder/layer_4/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_4/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_4/output/dense/bias" input: "bert/encoder/layer_4/output/dense/bias/adam_m" input: "bert/encoder/layer_4/output/dense/bias/adam_v" input: "bert/encoder/layer_4/output/dense/kernel" input: "bert/encoder/layer_4/output/dense/kernel/adam_m" input: "bert/encoder/layer_4/output/dense/kernel/adam_v" input: "bert/encoder/layer_5/attention/output/LayerNorm/beta" input: "bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_5/attention/output/LayerNorm/gamma" input: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_5/attention/output/dense/bias" input: "bert/encoder/layer_5/attention/output/dense/bias/adam_m" input: "bert/encoder/layer_5/attention/output/dense/bias/adam_v" input: "bert/encoder/layer_5/attention/output/dense/kernel" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v" input: "bert/encoder/layer_5/attention/self/key/bias" input: "bert/encoder/layer_5/attention/self/key/bias/adam_m" input: "bert/encoder/layer_5/attention/self/key/bias/adam_v" input: "bert/encoder/layer_5/attention/self/key/kernel" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_m" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_v" input: "bert/encoder/layer_5/attention/self/query/bias" input: "bert/encoder/layer_5/attention/self/query/bias/adam_m" input: "bert/encoder/layer_5/attention/self/query/bias/adam_v" input: "bert/encoder/layer_5/attention/self/query/kernel" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_m" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_v" input: "bert/encoder/layer_5/attention/self/value/bias" input: "bert/encoder/layer_5/attention/self/value/bias/adam_m" input: "bert/encoder/layer_5/attention/self/value/bias/adam_v" input: "bert/encoder/layer_5/attention/self/value/kernel" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_m" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_v" input: "bert/encoder/layer_5/intermediate/dense/bias" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_m" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_v" input: "bert/encoder/layer_5/intermediate/dense/kernel" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v" input: "bert/encoder/layer_5/output/LayerNorm/beta" input: "bert/encoder/layer_5/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_5/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_5/output/LayerNorm/gamma" input: "bert/encoder/layer_5/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_5/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_5/output/dense/bias" input: "bert/encoder/layer_5/output/dense/bias/adam_m" input: "bert/encoder/layer_5/output/dense/bias/adam_v" input: "bert/encoder/layer_5/output/dense/kernel" input: "bert/encoder/layer_5/output/dense/kernel/adam_m" input: "bert/encoder/layer_5/output/dense/kernel/adam_v" input: "bert/encoder/layer_6/attention/output/LayerNorm/beta" input: "bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_6/attention/output/LayerNorm/gamma" input: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_6/attention/output/dense/bias" input: "bert/encoder/layer_6/attention/output/dense/bias/adam_m" input: "bert/encoder/layer_6/attention/output/dense/bias/adam_v" input: "bert/encoder/layer_6/attention/output/dense/kernel" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v" input: "bert/encoder/layer_6/attention/self/key/bias" input: "bert/encoder/layer_6/attention/self/key/bias/adam_m" input: "bert/encoder/layer_6/attention/self/key/bias/adam_v" input: "bert/encoder/layer_6/attention/self/key/kernel" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_m" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_v" input: "bert/encoder/layer_6/attention/self/query/bias" input: "bert/encoder/layer_6/attention/self/query/bias/adam_m" input: "bert/encoder/layer_6/attention/self/query/bias/adam_v" input: "bert/encoder/layer_6/attention/self/query/kernel" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_m" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_v" input: "bert/encoder/layer_6/attention/self/value/bias" input: "bert/encoder/layer_6/attention/self/value/bias/adam_m" input: "bert/encoder/layer_6/attention/self/value/bias/adam_v" input: "bert/encoder/layer_6/attention/self/value/kernel" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_m" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_v" input: "bert/encoder/layer_6/intermediate/dense/bias" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_m" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_v" input: "bert/encoder/layer_6/intermediate/dense/kernel" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v" input: "bert/encoder/layer_6/output/LayerNorm/beta" input: "bert/encoder/layer_6/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_6/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_6/output/LayerNorm/gamma" input: "bert/encoder/layer_6/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_6/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_6/output/dense/bias" input: "bert/encoder/layer_6/output/dense/bias/adam_m" input: "bert/encoder/layer_6/output/dense/bias/adam_v" input: "bert/encoder/layer_6/output/dense/kernel" input: "bert/encoder/layer_6/output/dense/kernel/adam_m" input: "bert/encoder/layer_6/output/dense/kernel/adam_v" input: "bert/encoder/layer_7/attention/output/LayerNorm/beta" input: "bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_7/attention/output/LayerNorm/gamma" input: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_7/attention/output/dense/bias" input: "bert/encoder/layer_7/attention/output/dense/bias/adam_m" input: "bert/encoder/layer_7/attention/output/dense/bias/adam_v" input: "bert/encoder/layer_7/attention/output/dense/kernel" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v" input: "bert/encoder/layer_7/attention/self/key/bias" input: "bert/encoder/layer_7/attention/self/key/bias/adam_m" input: "bert/encoder/layer_7/attention/self/key/bias/adam_v" input: "bert/encoder/layer_7/attention/self/key/kernel" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_m" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_v" input: "bert/encoder/layer_7/attention/self/query/bias" input: "bert/encoder/layer_7/attention/self/query/bias/adam_m" input: "bert/encoder/layer_7/attention/self/query/bias/adam_v" input: "bert/encoder/layer_7/attention/self/query/kernel" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_m" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_v" input: "bert/encoder/layer_7/attention/self/value/bias" input: "bert/encoder/layer_7/attention/self/value/bias/adam_m" input: "bert/encoder/layer_7/attention/self/value/bias/adam_v" input: "bert/encoder/layer_7/attention/self/value/kernel" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_m" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_v" input: "bert/encoder/layer_7/intermediate/dense/bias" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_m" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_v" input: "bert/encoder/layer_7/intermediate/dense/kernel" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v" input: "bert/encoder/layer_7/output/LayerNorm/beta" input: "bert/encoder/layer_7/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_7/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_7/output/LayerNorm/gamma" input: "bert/encoder/layer_7/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_7/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_7/output/dense/bias" input: "bert/encoder/layer_7/output/dense/bias/adam_m" input: "bert/encoder/layer_7/output/dense/bias/adam_v" input: "bert/encoder/layer_7/output/dense/kernel" input: "bert/encoder/layer_7/output/dense/kernel/adam_m" input: "bert/encoder/layer_7/output/dense/kernel/adam_v" input: "bert/encoder/layer_8/attention/output/LayerNorm/beta" input: "bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_8/attention/output/LayerNorm/gamma" input: "bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_8/attention/output/dense/bias" input: "bert/encoder/layer_8/attention/output/dense/bias/adam_m" input: "bert/encoder/layer_8/attention/output/dense/bias/adam_v" input: "bert/encoder/layer_8/attention/output/dense/kernel" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v" input: "bert/encoder/layer_8/attention/self/key/bias" input: "bert/encoder/layer_8/attention/self/key/bias/adam_m" input: "bert/encoder/layer_8/attention/self/key/bias/adam_v" input: "bert/encoder/layer_8/attention/self/key/kernel" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_m" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_v" input: "bert/encoder/layer_8/attention/self/query/bias" input: "bert/encoder/layer_8/attention/self/query/bias/adam_m" input: "bert/encoder/layer_8/attention/self/query/bias/adam_v" input: "bert/encoder/layer_8/attention/self/query/kernel" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_m" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_v" input: "bert/encoder/layer_8/attention/self/value/bias" input: "bert/encoder/layer_8/attention/self/value/bias/adam_m" input: "bert/encoder/layer_8/attention/self/value/bias/adam_v" input: "bert/encoder/layer_8/attention/self/value/kernel" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_m" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_v" input: "bert/encoder/layer_8/intermediate/dense/bias" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_m" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_v" input: "bert/encoder/layer_8/intermediate/dense/kernel" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v" input: "bert/encoder/layer_8/output/LayerNorm/beta" input: "bert/encoder/layer_8/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_8/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_8/output/LayerNorm/gamma" input: "bert/encoder/layer_8/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_8/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_8/output/dense/bias" input: "bert/encoder/layer_8/output/dense/bias/adam_m" input: "bert/encoder/layer_8/output/dense/bias/adam_v" input: "bert/encoder/layer_8/output/dense/kernel" input: "bert/encoder/layer_8/output/dense/kernel/adam_m" input: "bert/encoder/layer_8/output/dense/kernel/adam_v" input: "bert/encoder/layer_9/attention/output/LayerNorm/beta" input: "bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_9/attention/output/LayerNorm/gamma" input: "bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_9/attention/output/dense/bias" input: "bert/encoder/layer_9/attention/output/dense/bias/adam_m" input: "bert/encoder/layer_9/attention/output/dense/bias/adam_v" input: "bert/encoder/layer_9/attention/output/dense/kernel" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v" input: "bert/encoder/layer_9/attention/self/key/bias" input: "bert/encoder/layer_9/attention/self/key/bias/adam_m" input: "bert/encoder/layer_9/attention/self/key/bias/adam_v" input: "bert/encoder/layer_9/attention/self/key/kernel" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_m" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_v" input: "bert/encoder/layer_9/attention/self/query/bias" input: "bert/encoder/layer_9/attention/self/query/bias/adam_m" input: "bert/encoder/layer_9/attention/self/query/bias/adam_v" input: "bert/encoder/layer_9/attention/self/query/kernel" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_m" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_v" input: "bert/encoder/layer_9/attention/self/value/bias" input: "bert/encoder/layer_9/attention/self/value/bias/adam_m" input: "bert/encoder/layer_9/attention/self/value/bias/adam_v" input: "bert/encoder/layer_9/attention/self/value/kernel" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_m" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_v" input: "bert/encoder/layer_9/intermediate/dense/bias" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_m" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_v" input: "bert/encoder/layer_9/intermediate/dense/kernel" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v" input: "bert/encoder/layer_9/output/LayerNorm/beta" input: "bert/encoder/layer_9/output/LayerNorm/beta/adam_m" input: "bert/encoder/layer_9/output/LayerNorm/beta/adam_v" input: "bert/encoder/layer_9/output/LayerNorm/gamma" input: "bert/encoder/layer_9/output/LayerNorm/gamma/adam_m" input: "bert/encoder/layer_9/output/LayerNorm/gamma/adam_v" input: "bert/encoder/layer_9/output/dense/bias" input: "bert/encoder/layer_9/output/dense/bias/adam_m" input: "bert/encoder/layer_9/output/dense/bias/adam_v" input: "bert/encoder/layer_9/output/dense/kernel" input: "bert/encoder/layer_9/output/dense/kernel/adam_m" input: "bert/encoder/layer_9/output/dense/kernel/adam_v" input: "bert/pooler/dense/bias" input: "bert/pooler/dense/bias/adam_m" input: "bert/pooler/dense/bias/adam_v" input: "bert/pooler/dense/kernel" input: "bert/pooler/dense/kernel/adam_m" input: "bert/pooler/dense/kernel/adam_v" input: "cls/predictions/output_bias" input: "cls/predictions/output_bias/adam_m" input: "cls/predictions/output_bias/adam_v" input: "cls/predictions/transform/LayerNorm/beta" input: "cls/predictions/transform/LayerNorm/beta/adam_m" input: "cls/predictions/transform/LayerNorm/beta/adam_v" input: "cls/predictions/transform/LayerNorm/gamma" input: "cls/predictions/transform/LayerNorm/gamma/adam_m" input: "cls/predictions/transform/LayerNorm/gamma/adam_v" input: "cls/predictions/transform/dense/bias" input: "cls/predictions/transform/dense/bias/adam_m" input: "cls/predictions/transform/dense/bias/adam_v" input: "cls/predictions/transform/dense/kernel" input: "cls/predictions/transform/dense/kernel/adam_m" input: "cls/predictions/transform/dense/kernel/adam_v" input: "cls/seq_relationship/output_bias" input: "cls/seq_relationship/output_bias/adam_m" input: "cls/seq_relationship/output_bias/adam_v" input: "cls/seq_relationship/output_weights" input: "cls/seq_relationship/output_weights/adam_m" input: "cls/seq_relationship/output_weights/adam_v" input: "global_step/Read/ReadVariableOp" device: "/device:CPU:0" attr { key: "dtypes" value { list { type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_INT64 } } } } node { name: "save/control_dependency" op: "Identity" input: "save/ShardedFilename" input: "^save/SaveV2" device: "/device:CPU:0" attr { key: "T" value { type: DT_STRING } } attr { key: "_class" value { list { s: "loc:@save/ShardedFilename" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "save/MergeV2Checkpoints/checkpoint_prefixes" op: "Pack" input: "save/ShardedFilename" input: "^save/control_dependency" device: "/device:CPU:0" attr { key: "N" value { i: 1 } } attr { key: "T" value { type: DT_STRING } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "axis" value { i: 0 } } } node { name: "save/MergeV2Checkpoints" op: "MergeV2Checkpoints" input: "save/MergeV2Checkpoints/checkpoint_prefixes" input: "save/Const" device: "/device:CPU:0" attr { key: "delete_old_dirs" value { b: true } } } node { name: "save/Identity" op: "Identity" input: "save/Const" input: "^save/MergeV2Checkpoints" input: "^save/control_dependency" device: "/device:CPU:0" attr { key: "T" value { type: DT_STRING } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "save/RestoreV2/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 619 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 619 } } string_val: "bert/embeddings/LayerNorm/beta" string_val: "bert/embeddings/LayerNorm/beta/adam_m" string_val: "bert/embeddings/LayerNorm/beta/adam_v" string_val: "bert/embeddings/LayerNorm/gamma" string_val: "bert/embeddings/LayerNorm/gamma/adam_m" string_val: "bert/embeddings/LayerNorm/gamma/adam_v" string_val: "bert/embeddings/position_embeddings" string_val: "bert/embeddings/position_embeddings/adam_m" string_val: "bert/embeddings/position_embeddings/adam_v" string_val: "bert/embeddings/token_type_embeddings" string_val: "bert/embeddings/token_type_embeddings/adam_m" string_val: "bert/embeddings/token_type_embeddings/adam_v" string_val: "bert/embeddings/word_embeddings" string_val: "bert/embeddings/word_embeddings/adam_m" string_val: "bert/embeddings/word_embeddings/adam_v" string_val: "bert/encoder/layer_0/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_0/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_0/attention/output/dense/bias" string_val: "bert/encoder/layer_0/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_0/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_0/attention/output/dense/kernel" string_val: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_0/attention/self/key/bias" string_val: "bert/encoder/layer_0/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_0/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_0/attention/self/key/kernel" string_val: "bert/encoder/layer_0/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_0/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_0/attention/self/query/bias" string_val: "bert/encoder/layer_0/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_0/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_0/attention/self/query/kernel" string_val: "bert/encoder/layer_0/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_0/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_0/attention/self/value/bias" string_val: "bert/encoder/layer_0/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_0/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_0/attention/self/value/kernel" string_val: "bert/encoder/layer_0/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_0/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_0/intermediate/dense/bias" string_val: "bert/encoder/layer_0/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_0/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_0/intermediate/dense/kernel" string_val: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_0/output/LayerNorm/beta" string_val: "bert/encoder/layer_0/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_0/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_0/output/LayerNorm/gamma" string_val: "bert/encoder/layer_0/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_0/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_0/output/dense/bias" string_val: "bert/encoder/layer_0/output/dense/bias/adam_m" string_val: "bert/encoder/layer_0/output/dense/bias/adam_v" string_val: "bert/encoder/layer_0/output/dense/kernel" string_val: "bert/encoder/layer_0/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_0/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_1/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_1/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_1/attention/output/dense/bias" string_val: "bert/encoder/layer_1/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_1/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_1/attention/output/dense/kernel" string_val: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_1/attention/self/key/bias" string_val: "bert/encoder/layer_1/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_1/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_1/attention/self/key/kernel" string_val: "bert/encoder/layer_1/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_1/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_1/attention/self/query/bias" string_val: "bert/encoder/layer_1/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_1/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_1/attention/self/query/kernel" string_val: "bert/encoder/layer_1/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_1/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_1/attention/self/value/bias" string_val: "bert/encoder/layer_1/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_1/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_1/attention/self/value/kernel" string_val: "bert/encoder/layer_1/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_1/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_1/intermediate/dense/bias" string_val: "bert/encoder/layer_1/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_1/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_1/intermediate/dense/kernel" string_val: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_1/output/LayerNorm/beta" string_val: "bert/encoder/layer_1/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_1/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_1/output/LayerNorm/gamma" string_val: "bert/encoder/layer_1/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_1/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_1/output/dense/bias" string_val: "bert/encoder/layer_1/output/dense/bias/adam_m" string_val: "bert/encoder/layer_1/output/dense/bias/adam_v" string_val: "bert/encoder/layer_1/output/dense/kernel" string_val: "bert/encoder/layer_1/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_1/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_10/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_10/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_10/attention/output/dense/bias" string_val: "bert/encoder/layer_10/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_10/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_10/attention/output/dense/kernel" string_val: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_10/attention/self/key/bias" string_val: "bert/encoder/layer_10/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_10/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_10/attention/self/key/kernel" string_val: "bert/encoder/layer_10/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_10/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_10/attention/self/query/bias" string_val: "bert/encoder/layer_10/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_10/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_10/attention/self/query/kernel" string_val: "bert/encoder/layer_10/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_10/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_10/attention/self/value/bias" string_val: "bert/encoder/layer_10/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_10/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_10/attention/self/value/kernel" string_val: "bert/encoder/layer_10/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_10/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_10/intermediate/dense/bias" string_val: "bert/encoder/layer_10/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_10/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_10/intermediate/dense/kernel" string_val: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_10/output/LayerNorm/beta" string_val: "bert/encoder/layer_10/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_10/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_10/output/LayerNorm/gamma" string_val: "bert/encoder/layer_10/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_10/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_10/output/dense/bias" string_val: "bert/encoder/layer_10/output/dense/bias/adam_m" string_val: "bert/encoder/layer_10/output/dense/bias/adam_v" string_val: "bert/encoder/layer_10/output/dense/kernel" string_val: "bert/encoder/layer_10/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_10/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_11/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_11/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_11/attention/output/dense/bias" string_val: "bert/encoder/layer_11/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_11/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_11/attention/output/dense/kernel" string_val: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_11/attention/self/key/bias" string_val: "bert/encoder/layer_11/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_11/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_11/attention/self/key/kernel" string_val: "bert/encoder/layer_11/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_11/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_11/attention/self/query/bias" string_val: "bert/encoder/layer_11/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_11/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_11/attention/self/query/kernel" string_val: "bert/encoder/layer_11/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_11/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_11/attention/self/value/bias" string_val: "bert/encoder/layer_11/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_11/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_11/attention/self/value/kernel" string_val: "bert/encoder/layer_11/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_11/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_11/intermediate/dense/bias" string_val: "bert/encoder/layer_11/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_11/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_11/intermediate/dense/kernel" string_val: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_11/output/LayerNorm/beta" string_val: "bert/encoder/layer_11/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_11/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_11/output/LayerNorm/gamma" string_val: "bert/encoder/layer_11/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_11/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_11/output/dense/bias" string_val: "bert/encoder/layer_11/output/dense/bias/adam_m" string_val: "bert/encoder/layer_11/output/dense/bias/adam_v" string_val: "bert/encoder/layer_11/output/dense/kernel" string_val: "bert/encoder/layer_11/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_11/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_2/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_2/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_2/attention/output/dense/bias" string_val: "bert/encoder/layer_2/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_2/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_2/attention/output/dense/kernel" string_val: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_2/attention/self/key/bias" string_val: "bert/encoder/layer_2/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_2/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_2/attention/self/key/kernel" string_val: "bert/encoder/layer_2/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_2/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_2/attention/self/query/bias" string_val: "bert/encoder/layer_2/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_2/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_2/attention/self/query/kernel" string_val: "bert/encoder/layer_2/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_2/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_2/attention/self/value/bias" string_val: "bert/encoder/layer_2/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_2/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_2/attention/self/value/kernel" string_val: "bert/encoder/layer_2/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_2/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_2/intermediate/dense/bias" string_val: "bert/encoder/layer_2/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_2/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_2/intermediate/dense/kernel" string_val: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_2/output/LayerNorm/beta" string_val: "bert/encoder/layer_2/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_2/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_2/output/LayerNorm/gamma" string_val: "bert/encoder/layer_2/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_2/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_2/output/dense/bias" string_val: "bert/encoder/layer_2/output/dense/bias/adam_m" string_val: "bert/encoder/layer_2/output/dense/bias/adam_v" string_val: "bert/encoder/layer_2/output/dense/kernel" string_val: "bert/encoder/layer_2/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_2/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_3/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_3/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_3/attention/output/dense/bias" string_val: "bert/encoder/layer_3/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_3/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_3/attention/output/dense/kernel" string_val: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_3/attention/self/key/bias" string_val: "bert/encoder/layer_3/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_3/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_3/attention/self/key/kernel" string_val: "bert/encoder/layer_3/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_3/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_3/attention/self/query/bias" string_val: "bert/encoder/layer_3/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_3/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_3/attention/self/query/kernel" string_val: "bert/encoder/layer_3/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_3/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_3/attention/self/value/bias" string_val: "bert/encoder/layer_3/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_3/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_3/attention/self/value/kernel" string_val: "bert/encoder/layer_3/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_3/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_3/intermediate/dense/bias" string_val: "bert/encoder/layer_3/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_3/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_3/intermediate/dense/kernel" string_val: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_3/output/LayerNorm/beta" string_val: "bert/encoder/layer_3/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_3/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_3/output/LayerNorm/gamma" string_val: "bert/encoder/layer_3/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_3/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_3/output/dense/bias" string_val: "bert/encoder/layer_3/output/dense/bias/adam_m" string_val: "bert/encoder/layer_3/output/dense/bias/adam_v" string_val: "bert/encoder/layer_3/output/dense/kernel" string_val: "bert/encoder/layer_3/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_3/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_4/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_4/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_4/attention/output/dense/bias" string_val: "bert/encoder/layer_4/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_4/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_4/attention/output/dense/kernel" string_val: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_4/attention/self/key/bias" string_val: "bert/encoder/layer_4/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_4/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_4/attention/self/key/kernel" string_val: "bert/encoder/layer_4/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_4/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_4/attention/self/query/bias" string_val: "bert/encoder/layer_4/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_4/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_4/attention/self/query/kernel" string_val: "bert/encoder/layer_4/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_4/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_4/attention/self/value/bias" string_val: "bert/encoder/layer_4/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_4/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_4/attention/self/value/kernel" string_val: "bert/encoder/layer_4/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_4/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_4/intermediate/dense/bias" string_val: "bert/encoder/layer_4/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_4/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_4/intermediate/dense/kernel" string_val: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_4/output/LayerNorm/beta" string_val: "bert/encoder/layer_4/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_4/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_4/output/LayerNorm/gamma" string_val: "bert/encoder/layer_4/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_4/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_4/output/dense/bias" string_val: "bert/encoder/layer_4/output/dense/bias/adam_m" string_val: "bert/encoder/layer_4/output/dense/bias/adam_v" string_val: "bert/encoder/layer_4/output/dense/kernel" string_val: "bert/encoder/layer_4/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_4/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_5/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_5/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_5/attention/output/dense/bias" string_val: "bert/encoder/layer_5/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_5/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_5/attention/output/dense/kernel" string_val: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_5/attention/self/key/bias" string_val: "bert/encoder/layer_5/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_5/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_5/attention/self/key/kernel" string_val: "bert/encoder/layer_5/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_5/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_5/attention/self/query/bias" string_val: "bert/encoder/layer_5/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_5/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_5/attention/self/query/kernel" string_val: "bert/encoder/layer_5/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_5/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_5/attention/self/value/bias" string_val: "bert/encoder/layer_5/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_5/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_5/attention/self/value/kernel" string_val: "bert/encoder/layer_5/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_5/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_5/intermediate/dense/bias" string_val: "bert/encoder/layer_5/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_5/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_5/intermediate/dense/kernel" string_val: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_5/output/LayerNorm/beta" string_val: "bert/encoder/layer_5/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_5/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_5/output/LayerNorm/gamma" string_val: "bert/encoder/layer_5/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_5/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_5/output/dense/bias" string_val: "bert/encoder/layer_5/output/dense/bias/adam_m" string_val: "bert/encoder/layer_5/output/dense/bias/adam_v" string_val: "bert/encoder/layer_5/output/dense/kernel" string_val: "bert/encoder/layer_5/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_5/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_6/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_6/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_6/attention/output/dense/bias" string_val: "bert/encoder/layer_6/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_6/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_6/attention/output/dense/kernel" string_val: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_6/attention/self/key/bias" string_val: "bert/encoder/layer_6/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_6/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_6/attention/self/key/kernel" string_val: "bert/encoder/layer_6/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_6/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_6/attention/self/query/bias" string_val: "bert/encoder/layer_6/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_6/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_6/attention/self/query/kernel" string_val: "bert/encoder/layer_6/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_6/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_6/attention/self/value/bias" string_val: "bert/encoder/layer_6/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_6/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_6/attention/self/value/kernel" string_val: "bert/encoder/layer_6/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_6/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_6/intermediate/dense/bias" string_val: "bert/encoder/layer_6/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_6/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_6/intermediate/dense/kernel" string_val: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_6/output/LayerNorm/beta" string_val: "bert/encoder/layer_6/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_6/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_6/output/LayerNorm/gamma" string_val: "bert/encoder/layer_6/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_6/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_6/output/dense/bias" string_val: "bert/encoder/layer_6/output/dense/bias/adam_m" string_val: "bert/encoder/layer_6/output/dense/bias/adam_v" string_val: "bert/encoder/layer_6/output/dense/kernel" string_val: "bert/encoder/layer_6/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_6/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_7/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_7/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_7/attention/output/dense/bias" string_val: "bert/encoder/layer_7/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_7/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_7/attention/output/dense/kernel" string_val: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_7/attention/self/key/bias" string_val: "bert/encoder/layer_7/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_7/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_7/attention/self/key/kernel" string_val: "bert/encoder/layer_7/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_7/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_7/attention/self/query/bias" string_val: "bert/encoder/layer_7/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_7/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_7/attention/self/query/kernel" string_val: "bert/encoder/layer_7/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_7/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_7/attention/self/value/bias" string_val: "bert/encoder/layer_7/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_7/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_7/attention/self/value/kernel" string_val: "bert/encoder/layer_7/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_7/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_7/intermediate/dense/bias" string_val: "bert/encoder/layer_7/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_7/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_7/intermediate/dense/kernel" string_val: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_7/output/LayerNorm/beta" string_val: "bert/encoder/layer_7/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_7/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_7/output/LayerNorm/gamma" string_val: "bert/encoder/layer_7/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_7/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_7/output/dense/bias" string_val: "bert/encoder/layer_7/output/dense/bias/adam_m" string_val: "bert/encoder/layer_7/output/dense/bias/adam_v" string_val: "bert/encoder/layer_7/output/dense/kernel" string_val: "bert/encoder/layer_7/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_7/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_8/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_8/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_8/attention/output/dense/bias" string_val: "bert/encoder/layer_8/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_8/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_8/attention/output/dense/kernel" string_val: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_8/attention/self/key/bias" string_val: "bert/encoder/layer_8/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_8/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_8/attention/self/key/kernel" string_val: "bert/encoder/layer_8/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_8/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_8/attention/self/query/bias" string_val: "bert/encoder/layer_8/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_8/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_8/attention/self/query/kernel" string_val: "bert/encoder/layer_8/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_8/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_8/attention/self/value/bias" string_val: "bert/encoder/layer_8/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_8/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_8/attention/self/value/kernel" string_val: "bert/encoder/layer_8/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_8/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_8/intermediate/dense/bias" string_val: "bert/encoder/layer_8/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_8/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_8/intermediate/dense/kernel" string_val: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_8/output/LayerNorm/beta" string_val: "bert/encoder/layer_8/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_8/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_8/output/LayerNorm/gamma" string_val: "bert/encoder/layer_8/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_8/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_8/output/dense/bias" string_val: "bert/encoder/layer_8/output/dense/bias/adam_m" string_val: "bert/encoder/layer_8/output/dense/bias/adam_v" string_val: "bert/encoder/layer_8/output/dense/kernel" string_val: "bert/encoder/layer_8/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_8/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_9/attention/output/LayerNorm/beta" string_val: "bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_9/attention/output/LayerNorm/gamma" string_val: "bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_9/attention/output/dense/bias" string_val: "bert/encoder/layer_9/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_9/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_9/attention/output/dense/kernel" string_val: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_9/attention/self/key/bias" string_val: "bert/encoder/layer_9/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_9/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_9/attention/self/key/kernel" string_val: "bert/encoder/layer_9/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_9/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_9/attention/self/query/bias" string_val: "bert/encoder/layer_9/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_9/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_9/attention/self/query/kernel" string_val: "bert/encoder/layer_9/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_9/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_9/attention/self/value/bias" string_val: "bert/encoder/layer_9/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_9/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_9/attention/self/value/kernel" string_val: "bert/encoder/layer_9/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_9/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_9/intermediate/dense/bias" string_val: "bert/encoder/layer_9/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_9/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_9/intermediate/dense/kernel" string_val: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_9/output/LayerNorm/beta" string_val: "bert/encoder/layer_9/output/LayerNorm/beta/adam_m" string_val: "bert/encoder/layer_9/output/LayerNorm/beta/adam_v" string_val: "bert/encoder/layer_9/output/LayerNorm/gamma" string_val: "bert/encoder/layer_9/output/LayerNorm/gamma/adam_m" string_val: "bert/encoder/layer_9/output/LayerNorm/gamma/adam_v" string_val: "bert/encoder/layer_9/output/dense/bias" string_val: "bert/encoder/layer_9/output/dense/bias/adam_m" string_val: "bert/encoder/layer_9/output/dense/bias/adam_v" string_val: "bert/encoder/layer_9/output/dense/kernel" string_val: "bert/encoder/layer_9/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_9/output/dense/kernel/adam_v" string_val: "bert/pooler/dense/bias" string_val: "bert/pooler/dense/bias/adam_m" string_val: "bert/pooler/dense/bias/adam_v" string_val: "bert/pooler/dense/kernel" string_val: "bert/pooler/dense/kernel/adam_m" string_val: "bert/pooler/dense/kernel/adam_v" string_val: "cls/predictions/output_bias" string_val: "cls/predictions/output_bias/adam_m" string_val: "cls/predictions/output_bias/adam_v" string_val: "cls/predictions/transform/LayerNorm/beta" string_val: "cls/predictions/transform/LayerNorm/beta/adam_m" string_val: "cls/predictions/transform/LayerNorm/beta/adam_v" string_val: "cls/predictions/transform/LayerNorm/gamma" string_val: "cls/predictions/transform/LayerNorm/gamma/adam_m" string_val: "cls/predictions/transform/LayerNorm/gamma/adam_v" string_val: "cls/predictions/transform/dense/bias" string_val: "cls/predictions/transform/dense/bias/adam_m" string_val: "cls/predictions/transform/dense/bias/adam_v" string_val: "cls/predictions/transform/dense/kernel" string_val: "cls/predictions/transform/dense/kernel/adam_m" string_val: "cls/predictions/transform/dense/kernel/adam_v" string_val: "cls/seq_relationship/output_bias" string_val: "cls/seq_relationship/output_bias/adam_m" string_val: "cls/seq_relationship/output_bias/adam_v" string_val: "cls/seq_relationship/output_weights" string_val: "cls/seq_relationship/output_weights/adam_m" string_val: "cls/seq_relationship/output_weights/adam_v" string_val: "global_step" } } } } node { name: "save/RestoreV2/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 619 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 619 } } string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" } } } } node { name: "save/RestoreV2" op: "RestoreV2" input: "save/Const" input: "save/RestoreV2/tensor_names" input: "save/RestoreV2/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } } } } attr { key: "dtypes" value { list { type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_INT64 } } } } node { name: "save/Assign" op: "Assign" input: "bert/embeddings/LayerNorm/beta" input: "save/RestoreV2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_1" op: "Assign" input: "bert/embeddings/LayerNorm/beta/adam_m" input: "save/RestoreV2:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_2" op: "Assign" input: "bert/embeddings/LayerNorm/beta/adam_v" input: "save/RestoreV2:2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_3" op: "Assign" input: "bert/embeddings/LayerNorm/gamma" input: "save/RestoreV2:3" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_4" op: "Assign" input: "bert/embeddings/LayerNorm/gamma/adam_m" input: "save/RestoreV2:4" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_5" op: "Assign" input: "bert/embeddings/LayerNorm/gamma/adam_v" input: "save/RestoreV2:5" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_6" op: "Assign" input: "bert/embeddings/position_embeddings" input: "save/RestoreV2:6" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_7" op: "Assign" input: "bert/embeddings/position_embeddings/adam_m" input: "save/RestoreV2:7" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_8" op: "Assign" input: "bert/embeddings/position_embeddings/adam_v" input: "save/RestoreV2:8" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_9" op: "Assign" input: "bert/embeddings/token_type_embeddings" input: "save/RestoreV2:9" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_10" op: "Assign" input: "bert/embeddings/token_type_embeddings/adam_m" input: "save/RestoreV2:10" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_11" op: "Assign" input: "bert/embeddings/token_type_embeddings/adam_v" input: "save/RestoreV2:11" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_12" op: "Assign" input: "bert/embeddings/word_embeddings" input: "save/RestoreV2:12" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_13" op: "Assign" input: "bert/embeddings/word_embeddings/adam_m" input: "save/RestoreV2:13" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_14" op: "Assign" input: "bert/embeddings/word_embeddings/adam_v" input: "save/RestoreV2:14" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_15" op: "Assign" input: "bert/encoder/layer_0/attention/output/LayerNorm/beta" input: "save/RestoreV2:15" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_16" op: "Assign" input: "bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_m" input: "save/RestoreV2:16" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_17" op: "Assign" input: "bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_v" input: "save/RestoreV2:17" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_18" op: "Assign" input: "bert/encoder/layer_0/attention/output/LayerNorm/gamma" input: "save/RestoreV2:18" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_19" op: "Assign" input: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m" input: "save/RestoreV2:19" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_20" op: "Assign" input: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v" input: "save/RestoreV2:20" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_21" op: "Assign" input: "bert/encoder/layer_0/attention/output/dense/bias" input: "save/RestoreV2:21" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_22" op: "Assign" input: "bert/encoder/layer_0/attention/output/dense/bias/adam_m" input: "save/RestoreV2:22" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_23" op: "Assign" input: "bert/encoder/layer_0/attention/output/dense/bias/adam_v" input: "save/RestoreV2:23" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_24" op: "Assign" input: "bert/encoder/layer_0/attention/output/dense/kernel" input: "save/RestoreV2:24" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_25" op: "Assign" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m" input: "save/RestoreV2:25" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_26" op: "Assign" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v" input: "save/RestoreV2:26" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_27" op: "Assign" input: "bert/encoder/layer_0/attention/self/key/bias" input: "save/RestoreV2:27" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_28" op: "Assign" input: "bert/encoder/layer_0/attention/self/key/bias/adam_m" input: "save/RestoreV2:28" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_29" op: "Assign" input: "bert/encoder/layer_0/attention/self/key/bias/adam_v" input: "save/RestoreV2:29" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_30" op: "Assign" input: "bert/encoder/layer_0/attention/self/key/kernel" input: "save/RestoreV2:30" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_31" op: "Assign" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_m" input: "save/RestoreV2:31" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_32" op: "Assign" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_v" input: "save/RestoreV2:32" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_33" op: "Assign" input: "bert/encoder/layer_0/attention/self/query/bias" input: "save/RestoreV2:33" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_34" op: "Assign" input: "bert/encoder/layer_0/attention/self/query/bias/adam_m" input: "save/RestoreV2:34" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_35" op: "Assign" input: "bert/encoder/layer_0/attention/self/query/bias/adam_v" input: "save/RestoreV2:35" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_36" op: "Assign" input: "bert/encoder/layer_0/attention/self/query/kernel" input: "save/RestoreV2:36" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_37" op: "Assign" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_m" input: "save/RestoreV2:37" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_38" op: "Assign" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_v" input: "save/RestoreV2:38" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_39" op: "Assign" input: "bert/encoder/layer_0/attention/self/value/bias" input: "save/RestoreV2:39" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_40" op: "Assign" input: "bert/encoder/layer_0/attention/self/value/bias/adam_m" input: "save/RestoreV2:40" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_41" op: "Assign" input: "bert/encoder/layer_0/attention/self/value/bias/adam_v" input: "save/RestoreV2:41" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_42" op: "Assign" input: "bert/encoder/layer_0/attention/self/value/kernel" input: "save/RestoreV2:42" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_43" op: "Assign" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_m" input: "save/RestoreV2:43" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_44" op: "Assign" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_v" input: "save/RestoreV2:44" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_45" op: "Assign" input: "bert/encoder/layer_0/intermediate/dense/bias" input: "save/RestoreV2:45" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_46" op: "Assign" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_m" input: "save/RestoreV2:46" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_47" op: "Assign" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_v" input: "save/RestoreV2:47" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_48" op: "Assign" input: "bert/encoder/layer_0/intermediate/dense/kernel" input: "save/RestoreV2:48" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_49" op: "Assign" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m" input: "save/RestoreV2:49" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_50" op: "Assign" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v" input: "save/RestoreV2:50" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_51" op: "Assign" input: "bert/encoder/layer_0/output/LayerNorm/beta" input: "save/RestoreV2:51" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_52" op: "Assign" input: "bert/encoder/layer_0/output/LayerNorm/beta/adam_m" input: "save/RestoreV2:52" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_53" op: "Assign" input: "bert/encoder/layer_0/output/LayerNorm/beta/adam_v" input: "save/RestoreV2:53" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_54" op: "Assign" input: "bert/encoder/layer_0/output/LayerNorm/gamma" input: "save/RestoreV2:54" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_55" op: "Assign" input: "bert/encoder/layer_0/output/LayerNorm/gamma/adam_m" input: "save/RestoreV2:55" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_56" op: "Assign" input: "bert/encoder/layer_0/output/LayerNorm/gamma/adam_v" input: "save/RestoreV2:56" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_57" op: "Assign" input: "bert/encoder/layer_0/output/dense/bias" input: "save/RestoreV2:57" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_58" op: "Assign" input: "bert/encoder/layer_0/output/dense/bias/adam_m" input: "save/RestoreV2:58" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_59" op: "Assign" input: "bert/encoder/layer_0/output/dense/bias/adam_v" input: "save/RestoreV2:59" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_60" op: "Assign" input: "bert/encoder/layer_0/output/dense/kernel" input: "save/RestoreV2:60" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_61" op: "Assign" input: "bert/encoder/layer_0/output/dense/kernel/adam_m" input: "save/RestoreV2:61" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_62" op: "Assign" input: "bert/encoder/layer_0/output/dense/kernel/adam_v" input: "save/RestoreV2:62" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_63" op: "Assign" input: "bert/encoder/layer_1/attention/output/LayerNorm/beta" input: "save/RestoreV2:63" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_64" op: "Assign" input: "bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_m" input: "save/RestoreV2:64" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_65" op: "Assign" input: "bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_v" input: "save/RestoreV2:65" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_66" op: "Assign" input: "bert/encoder/layer_1/attention/output/LayerNorm/gamma" input: "save/RestoreV2:66" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_67" op: "Assign" input: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m" input: "save/RestoreV2:67" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_68" op: "Assign" input: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v" input: "save/RestoreV2:68" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_69" op: "Assign" input: "bert/encoder/layer_1/attention/output/dense/bias" input: "save/RestoreV2:69" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_70" op: "Assign" input: "bert/encoder/layer_1/attention/output/dense/bias/adam_m" input: "save/RestoreV2:70" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_71" op: "Assign" input: "bert/encoder/layer_1/attention/output/dense/bias/adam_v" input: "save/RestoreV2:71" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_72" op: "Assign" input: "bert/encoder/layer_1/attention/output/dense/kernel" input: "save/RestoreV2:72" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_73" op: "Assign" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m" input: "save/RestoreV2:73" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_74" op: "Assign" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v" input: "save/RestoreV2:74" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_75" op: "Assign" input: "bert/encoder/layer_1/attention/self/key/bias" input: "save/RestoreV2:75" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_76" op: "Assign" input: "bert/encoder/layer_1/attention/self/key/bias/adam_m" input: "save/RestoreV2:76" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_77" op: "Assign" input: "bert/encoder/layer_1/attention/self/key/bias/adam_v" input: "save/RestoreV2:77" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_78" op: "Assign" input: "bert/encoder/layer_1/attention/self/key/kernel" input: "save/RestoreV2:78" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_79" op: "Assign" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_m" input: "save/RestoreV2:79" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_80" op: "Assign" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_v" input: "save/RestoreV2:80" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_81" op: "Assign" input: "bert/encoder/layer_1/attention/self/query/bias" input: "save/RestoreV2:81" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_82" op: "Assign" input: "bert/encoder/layer_1/attention/self/query/bias/adam_m" input: "save/RestoreV2:82" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_83" op: "Assign" input: "bert/encoder/layer_1/attention/self/query/bias/adam_v" input: "save/RestoreV2:83" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_84" op: "Assign" input: "bert/encoder/layer_1/attention/self/query/kernel" input: "save/RestoreV2:84" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_85" op: "Assign" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_m" input: "save/RestoreV2:85" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_86" op: "Assign" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_v" input: "save/RestoreV2:86" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_87" op: "Assign" input: "bert/encoder/layer_1/attention/self/value/bias" input: "save/RestoreV2:87" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_88" op: "Assign" input: "bert/encoder/layer_1/attention/self/value/bias/adam_m" input: "save/RestoreV2:88" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_89" op: "Assign" input: "bert/encoder/layer_1/attention/self/value/bias/adam_v" input: "save/RestoreV2:89" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_90" op: "Assign" input: "bert/encoder/layer_1/attention/self/value/kernel" input: "save/RestoreV2:90" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_91" op: "Assign" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_m" input: "save/RestoreV2:91" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_92" op: "Assign" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_v" input: "save/RestoreV2:92" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_93" op: "Assign" input: "bert/encoder/layer_1/intermediate/dense/bias" input: "save/RestoreV2:93" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_94" op: "Assign" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_m" input: "save/RestoreV2:94" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_95" op: "Assign" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_v" input: "save/RestoreV2:95" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_96" op: "Assign" input: "bert/encoder/layer_1/intermediate/dense/kernel" input: "save/RestoreV2:96" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_97" op: "Assign" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m" input: "save/RestoreV2:97" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_98" op: "Assign" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v" input: "save/RestoreV2:98" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_99" op: "Assign" input: "bert/encoder/layer_1/output/LayerNorm/beta" input: "save/RestoreV2:99" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_100" op: "Assign" input: "bert/encoder/layer_1/output/LayerNorm/beta/adam_m" input: "save/RestoreV2:100" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_101" op: "Assign" input: "bert/encoder/layer_1/output/LayerNorm/beta/adam_v" input: "save/RestoreV2:101" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_102" op: "Assign" input: "bert/encoder/layer_1/output/LayerNorm/gamma" input: "save/RestoreV2:102" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_103" op: "Assign" input: "bert/encoder/layer_1/output/LayerNorm/gamma/adam_m" input: "save/RestoreV2:103" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_104" op: "Assign" input: "bert/encoder/layer_1/output/LayerNorm/gamma/adam_v" input: "save/RestoreV2:104" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_105" op: "Assign" input: "bert/encoder/layer_1/output/dense/bias" input: "save/RestoreV2:105" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_106" op: "Assign" input: "bert/encoder/layer_1/output/dense/bias/adam_m" input: "save/RestoreV2:106" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_107" op: "Assign" input: "bert/encoder/layer_1/output/dense/bias/adam_v" input: "save/RestoreV2:107" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_108" op: "Assign" input: "bert/encoder/layer_1/output/dense/kernel" input: "save/RestoreV2:108" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_109" op: "Assign" input: "bert/encoder/layer_1/output/dense/kernel/adam_m" input: "save/RestoreV2:109" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_110" op: "Assign" input: "bert/encoder/layer_1/output/dense/kernel/adam_v" input: "save/RestoreV2:110" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_111" op: "Assign" input: "bert/encoder/layer_10/attention/output/LayerNorm/beta" input: "save/RestoreV2:111" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_112" op: "Assign" input: "bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_m" input: "save/RestoreV2:112" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_113" op: "Assign" input: "bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_v" input: "save/RestoreV2:113" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_114" op: "Assign" input: "bert/encoder/layer_10/attention/output/LayerNorm/gamma" input: "save/RestoreV2:114" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_115" op: "Assign" input: "bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m" input: "save/RestoreV2:115" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_116" op: "Assign" input: "bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v" input: "save/RestoreV2:116" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_117" op: "Assign" input: "bert/encoder/layer_10/attention/output/dense/bias" input: "save/RestoreV2:117" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_118" op: "Assign" input: "bert/encoder/layer_10/attention/output/dense/bias/adam_m" input: "save/RestoreV2:118" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_119" op: "Assign" input: "bert/encoder/layer_10/attention/output/dense/bias/adam_v" input: "save/RestoreV2:119" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_120" op: "Assign" input: "bert/encoder/layer_10/attention/output/dense/kernel" input: "save/RestoreV2:120" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_121" op: "Assign" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m" input: "save/RestoreV2:121" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_122" op: "Assign" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v" input: "save/RestoreV2:122" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_123" op: "Assign" input: "bert/encoder/layer_10/attention/self/key/bias" input: "save/RestoreV2:123" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_124" op: "Assign" input: "bert/encoder/layer_10/attention/self/key/bias/adam_m" input: "save/RestoreV2:124" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_125" op: "Assign" input: "bert/encoder/layer_10/attention/self/key/bias/adam_v" input: "save/RestoreV2:125" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_126" op: "Assign" input: "bert/encoder/layer_10/attention/self/key/kernel" input: "save/RestoreV2:126" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_127" op: "Assign" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_m" input: "save/RestoreV2:127" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_128" op: "Assign" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_v" input: "save/RestoreV2:128" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_129" op: "Assign" input: "bert/encoder/layer_10/attention/self/query/bias" input: "save/RestoreV2:129" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_130" op: "Assign" input: "bert/encoder/layer_10/attention/self/query/bias/adam_m" input: "save/RestoreV2:130" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_131" op: "Assign" input: "bert/encoder/layer_10/attention/self/query/bias/adam_v" input: "save/RestoreV2:131" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_132" op: "Assign" input: "bert/encoder/layer_10/attention/self/query/kernel" input: "save/RestoreV2:132" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_133" op: "Assign" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_m" input: "save/RestoreV2:133" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_134" op: "Assign" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_v" input: "save/RestoreV2:134" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_135" op: "Assign" input: "bert/encoder/layer_10/attention/self/value/bias" input: "save/RestoreV2:135" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_136" op: "Assign" input: "bert/encoder/layer_10/attention/self/value/bias/adam_m" input: "save/RestoreV2:136" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_137" op: "Assign" input: "bert/encoder/layer_10/attention/self/value/bias/adam_v" input: "save/RestoreV2:137" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_138" op: "Assign" input: "bert/encoder/layer_10/attention/self/value/kernel" input: "save/RestoreV2:138" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_139" op: "Assign" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_m" input: "save/RestoreV2:139" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_140" op: "Assign" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_v" input: "save/RestoreV2:140" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_141" op: "Assign" input: "bert/encoder/layer_10/intermediate/dense/bias" input: "save/RestoreV2:141" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_142" op: "Assign" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_m" input: "save/RestoreV2:142" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_143" op: "Assign" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_v" input: "save/RestoreV2:143" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_144" op: "Assign" input: "bert/encoder/layer_10/intermediate/dense/kernel" input: "save/RestoreV2:144" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_145" op: "Assign" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m" input: "save/RestoreV2:145" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_146" op: "Assign" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v" input: "save/RestoreV2:146" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_147" op: "Assign" input: "bert/encoder/layer_10/output/LayerNorm/beta" input: "save/RestoreV2:147" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_148" op: "Assign" input: "bert/encoder/layer_10/output/LayerNorm/beta/adam_m" input: "save/RestoreV2:148" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_149" op: "Assign" input: "bert/encoder/layer_10/output/LayerNorm/beta/adam_v" input: "save/RestoreV2:149" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_150" op: "Assign" input: "bert/encoder/layer_10/output/LayerNorm/gamma" input: "save/RestoreV2:150" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_151" op: "Assign" input: "bert/encoder/layer_10/output/LayerNorm/gamma/adam_m" input: "save/RestoreV2:151" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_152" op: "Assign" input: "bert/encoder/layer_10/output/LayerNorm/gamma/adam_v" input: "save/RestoreV2:152" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_153" op: "Assign" input: "bert/encoder/layer_10/output/dense/bias" input: "save/RestoreV2:153" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_154" op: "Assign" input: "bert/encoder/layer_10/output/dense/bias/adam_m" input: "save/RestoreV2:154" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_155" op: "Assign" input: "bert/encoder/layer_10/output/dense/bias/adam_v" input: "save/RestoreV2:155" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_156" op: "Assign" input: "bert/encoder/layer_10/output/dense/kernel" input: "save/RestoreV2:156" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_157" op: "Assign" input: "bert/encoder/layer_10/output/dense/kernel/adam_m" input: "save/RestoreV2:157" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_158" op: "Assign" input: "bert/encoder/layer_10/output/dense/kernel/adam_v" input: "save/RestoreV2:158" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_159" op: "Assign" input: "bert/encoder/layer_11/attention/output/LayerNorm/beta" input: "save/RestoreV2:159" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_160" op: "Assign" input: "bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_m" input: "save/RestoreV2:160" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_161" op: "Assign" input: "bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_v" input: "save/RestoreV2:161" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_162" op: "Assign" input: "bert/encoder/layer_11/attention/output/LayerNorm/gamma" input: "save/RestoreV2:162" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_163" op: "Assign" input: "bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m" input: "save/RestoreV2:163" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_164" op: "Assign" input: "bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v" input: "save/RestoreV2:164" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_165" op: "Assign" input: "bert/encoder/layer_11/attention/output/dense/bias" input: "save/RestoreV2:165" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_166" op: "Assign" input: "bert/encoder/layer_11/attention/output/dense/bias/adam_m" input: "save/RestoreV2:166" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_167" op: "Assign" input: "bert/encoder/layer_11/attention/output/dense/bias/adam_v" input: "save/RestoreV2:167" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_168" op: "Assign" input: "bert/encoder/layer_11/attention/output/dense/kernel" input: "save/RestoreV2:168" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_169" op: "Assign" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m" input: "save/RestoreV2:169" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_170" op: "Assign" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v" input: "save/RestoreV2:170" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_171" op: "Assign" input: "bert/encoder/layer_11/attention/self/key/bias" input: "save/RestoreV2:171" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_172" op: "Assign" input: "bert/encoder/layer_11/attention/self/key/bias/adam_m" input: "save/RestoreV2:172" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_173" op: "Assign" input: "bert/encoder/layer_11/attention/self/key/bias/adam_v" input: "save/RestoreV2:173" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_174" op: "Assign" input: "bert/encoder/layer_11/attention/self/key/kernel" input: "save/RestoreV2:174" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_175" op: "Assign" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_m" input: "save/RestoreV2:175" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_176" op: "Assign" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_v" input: "save/RestoreV2:176" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_177" op: "Assign" input: "bert/encoder/layer_11/attention/self/query/bias" input: "save/RestoreV2:177" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_178" op: "Assign" input: "bert/encoder/layer_11/attention/self/query/bias/adam_m" input: "save/RestoreV2:178" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_179" op: "Assign" input: "bert/encoder/layer_11/attention/self/query/bias/adam_v" input: "save/RestoreV2:179" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_180" op: "Assign" input: "bert/encoder/layer_11/attention/self/query/kernel" input: "save/RestoreV2:180" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_181" op: "Assign" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_m" input: "save/RestoreV2:181" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_182" op: "Assign" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_v" input: "save/RestoreV2:182" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_183" op: "Assign" input: "bert/encoder/layer_11/attention/self/value/bias" input: "save/RestoreV2:183" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_184" op: "Assign" input: "bert/encoder/layer_11/attention/self/value/bias/adam_m" input: "save/RestoreV2:184" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_185" op: "Assign" input: "bert/encoder/layer_11/attention/self/value/bias/adam_v" input: "save/RestoreV2:185" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_186" op: "Assign" input: "bert/encoder/layer_11/attention/self/value/kernel" input: "save/RestoreV2:186" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_187" op: "Assign" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_m" input: "save/RestoreV2:187" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_188" op: "Assign" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_v" input: "save/RestoreV2:188" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_189" op: "Assign" input: "bert/encoder/layer_11/intermediate/dense/bias" input: "save/RestoreV2:189" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_190" op: "Assign" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_m" input: "save/RestoreV2:190" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_191" op: "Assign" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_v" input: "save/RestoreV2:191" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_192" op: "Assign" input: "bert/encoder/layer_11/intermediate/dense/kernel" input: "save/RestoreV2:192" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_193" op: "Assign" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m" input: "save/RestoreV2:193" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_194" op: "Assign" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v" input: "save/RestoreV2:194" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_195" op: "Assign" input: "bert/encoder/layer_11/output/LayerNorm/beta" input: "save/RestoreV2:195" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_196" op: "Assign" input: "bert/encoder/layer_11/output/LayerNorm/beta/adam_m" input: "save/RestoreV2:196" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_197" op: "Assign" input: "bert/encoder/layer_11/output/LayerNorm/beta/adam_v" input: "save/RestoreV2:197" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_198" op: "Assign" input: "bert/encoder/layer_11/output/LayerNorm/gamma" input: "save/RestoreV2:198" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_199" op: "Assign" input: "bert/encoder/layer_11/output/LayerNorm/gamma/adam_m" input: "save/RestoreV2:199" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_200" op: "Assign" input: "bert/encoder/layer_11/output/LayerNorm/gamma/adam_v" input: "save/RestoreV2:200" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_201" op: "Assign" input: "bert/encoder/layer_11/output/dense/bias" input: "save/RestoreV2:201" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_202" op: "Assign" input: "bert/encoder/layer_11/output/dense/bias/adam_m" input: "save/RestoreV2:202" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_203" op: "Assign" input: "bert/encoder/layer_11/output/dense/bias/adam_v" input: "save/RestoreV2:203" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_204" op: "Assign" input: "bert/encoder/layer_11/output/dense/kernel" input: "save/RestoreV2:204" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_205" op: "Assign" input: "bert/encoder/layer_11/output/dense/kernel/adam_m" input: "save/RestoreV2:205" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_206" op: "Assign" input: "bert/encoder/layer_11/output/dense/kernel/adam_v" input: "save/RestoreV2:206" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_207" op: "Assign" input: "bert/encoder/layer_2/attention/output/LayerNorm/beta" input: "save/RestoreV2:207" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_208" op: "Assign" input: "bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_m" input: "save/RestoreV2:208" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_209" op: "Assign" input: "bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_v" input: "save/RestoreV2:209" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_210" op: "Assign" input: "bert/encoder/layer_2/attention/output/LayerNorm/gamma" input: "save/RestoreV2:210" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_211" op: "Assign" input: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m" input: "save/RestoreV2:211" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_212" op: "Assign" input: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v" input: "save/RestoreV2:212" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_213" op: "Assign" input: "bert/encoder/layer_2/attention/output/dense/bias" input: "save/RestoreV2:213" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_214" op: "Assign" input: "bert/encoder/layer_2/attention/output/dense/bias/adam_m" input: "save/RestoreV2:214" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_215" op: "Assign" input: "bert/encoder/layer_2/attention/output/dense/bias/adam_v" input: "save/RestoreV2:215" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_216" op: "Assign" input: "bert/encoder/layer_2/attention/output/dense/kernel" input: "save/RestoreV2:216" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_217" op: "Assign" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m" input: "save/RestoreV2:217" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_218" op: "Assign" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v" input: "save/RestoreV2:218" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_219" op: "Assign" input: "bert/encoder/layer_2/attention/self/key/bias" input: "save/RestoreV2:219" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_220" op: "Assign" input: "bert/encoder/layer_2/attention/self/key/bias/adam_m" input: "save/RestoreV2:220" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_221" op: "Assign" input: "bert/encoder/layer_2/attention/self/key/bias/adam_v" input: "save/RestoreV2:221" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_222" op: "Assign" input: "bert/encoder/layer_2/attention/self/key/kernel" input: "save/RestoreV2:222" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_223" op: "Assign" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_m" input: "save/RestoreV2:223" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_224" op: "Assign" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_v" input: "save/RestoreV2:224" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_225" op: "Assign" input: "bert/encoder/layer_2/attention/self/query/bias" input: "save/RestoreV2:225" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_226" op: "Assign" input: "bert/encoder/layer_2/attention/self/query/bias/adam_m" input: "save/RestoreV2:226" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_227" op: "Assign" input: "bert/encoder/layer_2/attention/self/query/bias/adam_v" input: "save/RestoreV2:227" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_228" op: "Assign" input: "bert/encoder/layer_2/attention/self/query/kernel" input: "save/RestoreV2:228" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_229" op: "Assign" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_m" input: "save/RestoreV2:229" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_230" op: "Assign" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_v" input: "save/RestoreV2:230" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_231" op: "Assign" input: "bert/encoder/layer_2/attention/self/value/bias" input: "save/RestoreV2:231" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_232" op: "Assign" input: "bert/encoder/layer_2/attention/self/value/bias/adam_m" input: "save/RestoreV2:232" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_233" op: "Assign" input: "bert/encoder/layer_2/attention/self/value/bias/adam_v" input: "save/RestoreV2:233" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_234" op: "Assign" input: "bert/encoder/layer_2/attention/self/value/kernel" input: "save/RestoreV2:234" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_235" op: "Assign" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_m" input: "save/RestoreV2:235" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_236" op: "Assign" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_v" input: "save/RestoreV2:236" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_237" op: "Assign" input: "bert/encoder/layer_2/intermediate/dense/bias" input: "save/RestoreV2:237" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_238" op: "Assign" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_m" input: "save/RestoreV2:238" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_239" op: "Assign" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_v" input: "save/RestoreV2:239" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_240" op: "Assign" input: "bert/encoder/layer_2/intermediate/dense/kernel" input: "save/RestoreV2:240" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_241" op: "Assign" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m" input: "save/RestoreV2:241" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_242" op: "Assign" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v" input: "save/RestoreV2:242" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_243" op: "Assign" input: "bert/encoder/layer_2/output/LayerNorm/beta" input: "save/RestoreV2:243" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_244" op: "Assign" input: "bert/encoder/layer_2/output/LayerNorm/beta/adam_m" input: "save/RestoreV2:244" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_245" op: "Assign" input: "bert/encoder/layer_2/output/LayerNorm/beta/adam_v" input: "save/RestoreV2:245" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_246" op: "Assign" input: "bert/encoder/layer_2/output/LayerNorm/gamma" input: "save/RestoreV2:246" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_247" op: "Assign" input: "bert/encoder/layer_2/output/LayerNorm/gamma/adam_m" input: "save/RestoreV2:247" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_248" op: "Assign" input: "bert/encoder/layer_2/output/LayerNorm/gamma/adam_v" input: "save/RestoreV2:248" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_249" op: "Assign" input: "bert/encoder/layer_2/output/dense/bias" input: "save/RestoreV2:249" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_250" op: "Assign" input: "bert/encoder/layer_2/output/dense/bias/adam_m" input: "save/RestoreV2:250" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_251" op: "Assign" input: "bert/encoder/layer_2/output/dense/bias/adam_v" input: "save/RestoreV2:251" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_252" op: "Assign" input: "bert/encoder/layer_2/output/dense/kernel" input: "save/RestoreV2:252" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_253" op: "Assign" input: "bert/encoder/layer_2/output/dense/kernel/adam_m" input: "save/RestoreV2:253" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_254" op: "Assign" input: "bert/encoder/layer_2/output/dense/kernel/adam_v" input: "save/RestoreV2:254" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_255" op: "Assign" input: "bert/encoder/layer_3/attention/output/LayerNorm/beta" input: "save/RestoreV2:255" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_256" op: "Assign" input: "bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_m" input: "save/RestoreV2:256" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_257" op: "Assign" input: "bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_v" input: "save/RestoreV2:257" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_258" op: "Assign" input: "bert/encoder/layer_3/attention/output/LayerNorm/gamma" input: "save/RestoreV2:258" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_259" op: "Assign" input: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m" input: "save/RestoreV2:259" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_260" op: "Assign" input: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v" input: "save/RestoreV2:260" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_261" op: "Assign" input: "bert/encoder/layer_3/attention/output/dense/bias" input: "save/RestoreV2:261" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_262" op: "Assign" input: "bert/encoder/layer_3/attention/output/dense/bias/adam_m" input: "save/RestoreV2:262" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_263" op: "Assign" input: "bert/encoder/layer_3/attention/output/dense/bias/adam_v" input: "save/RestoreV2:263" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_264" op: "Assign" input: "bert/encoder/layer_3/attention/output/dense/kernel" input: "save/RestoreV2:264" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_265" op: "Assign" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m" input: "save/RestoreV2:265" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_266" op: "Assign" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v" input: "save/RestoreV2:266" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_267" op: "Assign" input: "bert/encoder/layer_3/attention/self/key/bias" input: "save/RestoreV2:267" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_268" op: "Assign" input: "bert/encoder/layer_3/attention/self/key/bias/adam_m" input: "save/RestoreV2:268" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_269" op: "Assign" input: "bert/encoder/layer_3/attention/self/key/bias/adam_v" input: "save/RestoreV2:269" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_270" op: "Assign" input: "bert/encoder/layer_3/attention/self/key/kernel" input: "save/RestoreV2:270" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_271" op: "Assign" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_m" input: "save/RestoreV2:271" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_272" op: "Assign" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_v" input: "save/RestoreV2:272" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_273" op: "Assign" input: "bert/encoder/layer_3/attention/self/query/bias" input: "save/RestoreV2:273" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_274" op: "Assign" input: "bert/encoder/layer_3/attention/self/query/bias/adam_m" input: "save/RestoreV2:274" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_275" op: "Assign" input: "bert/encoder/layer_3/attention/self/query/bias/adam_v" input: "save/RestoreV2:275" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_276" op: "Assign" input: "bert/encoder/layer_3/attention/self/query/kernel" input: "save/RestoreV2:276" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_277" op: "Assign" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_m" input: "save/RestoreV2:277" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_278" op: "Assign" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_v" input: "save/RestoreV2:278" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_279" op: "Assign" input: "bert/encoder/layer_3/attention/self/value/bias" input: "save/RestoreV2:279" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_280" op: "Assign" input: "bert/encoder/layer_3/attention/self/value/bias/adam_m" input: "save/RestoreV2:280" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_281" op: "Assign" input: "bert/encoder/layer_3/attention/self/value/bias/adam_v" input: "save/RestoreV2:281" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_282" op: "Assign" input: "bert/encoder/layer_3/attention/self/value/kernel" input: "save/RestoreV2:282" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_283" op: "Assign" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_m" input: "save/RestoreV2:283" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_284" op: "Assign" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_v" input: "save/RestoreV2:284" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_285" op: "Assign" input: "bert/encoder/layer_3/intermediate/dense/bias" input: "save/RestoreV2:285" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_286" op: "Assign" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_m" input: "save/RestoreV2:286" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_287" op: "Assign" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_v" input: "save/RestoreV2:287" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_288" op: "Assign" input: "bert/encoder/layer_3/intermediate/dense/kernel" input: "save/RestoreV2:288" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_289" op: "Assign" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m" input: "save/RestoreV2:289" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_290" op: "Assign" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v" input: "save/RestoreV2:290" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_291" op: "Assign" input: "bert/encoder/layer_3/output/LayerNorm/beta" input: "save/RestoreV2:291" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_292" op: "Assign" input: "bert/encoder/layer_3/output/LayerNorm/beta/adam_m" input: "save/RestoreV2:292" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_293" op: "Assign" input: "bert/encoder/layer_3/output/LayerNorm/beta/adam_v" input: "save/RestoreV2:293" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_294" op: "Assign" input: "bert/encoder/layer_3/output/LayerNorm/gamma" input: "save/RestoreV2:294" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_295" op: "Assign" input: "bert/encoder/layer_3/output/LayerNorm/gamma/adam_m" input: "save/RestoreV2:295" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_296" op: "Assign" input: "bert/encoder/layer_3/output/LayerNorm/gamma/adam_v" input: "save/RestoreV2:296" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_297" op: "Assign" input: "bert/encoder/layer_3/output/dense/bias" input: "save/RestoreV2:297" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_298" op: "Assign" input: "bert/encoder/layer_3/output/dense/bias/adam_m" input: "save/RestoreV2:298" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_299" op: "Assign" input: "bert/encoder/layer_3/output/dense/bias/adam_v" input: "save/RestoreV2:299" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_300" op: "Assign" input: "bert/encoder/layer_3/output/dense/kernel" input: "save/RestoreV2:300" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_301" op: "Assign" input: "bert/encoder/layer_3/output/dense/kernel/adam_m" input: "save/RestoreV2:301" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_302" op: "Assign" input: "bert/encoder/layer_3/output/dense/kernel/adam_v" input: "save/RestoreV2:302" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_303" op: "Assign" input: "bert/encoder/layer_4/attention/output/LayerNorm/beta" input: "save/RestoreV2:303" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_304" op: "Assign" input: "bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_m" input: "save/RestoreV2:304" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_305" op: "Assign" input: "bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_v" input: "save/RestoreV2:305" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_306" op: "Assign" input: "bert/encoder/layer_4/attention/output/LayerNorm/gamma" input: "save/RestoreV2:306" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_307" op: "Assign" input: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m" input: "save/RestoreV2:307" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_308" op: "Assign" input: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v" input: "save/RestoreV2:308" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_309" op: "Assign" input: "bert/encoder/layer_4/attention/output/dense/bias" input: "save/RestoreV2:309" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_310" op: "Assign" input: "bert/encoder/layer_4/attention/output/dense/bias/adam_m" input: "save/RestoreV2:310" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_311" op: "Assign" input: "bert/encoder/layer_4/attention/output/dense/bias/adam_v" input: "save/RestoreV2:311" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_312" op: "Assign" input: "bert/encoder/layer_4/attention/output/dense/kernel" input: "save/RestoreV2:312" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_313" op: "Assign" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m" input: "save/RestoreV2:313" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_314" op: "Assign" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v" input: "save/RestoreV2:314" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_315" op: "Assign" input: "bert/encoder/layer_4/attention/self/key/bias" input: "save/RestoreV2:315" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_316" op: "Assign" input: "bert/encoder/layer_4/attention/self/key/bias/adam_m" input: "save/RestoreV2:316" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_317" op: "Assign" input: "bert/encoder/layer_4/attention/self/key/bias/adam_v" input: "save/RestoreV2:317" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_318" op: "Assign" input: "bert/encoder/layer_4/attention/self/key/kernel" input: "save/RestoreV2:318" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_319" op: "Assign" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_m" input: "save/RestoreV2:319" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_320" op: "Assign" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_v" input: "save/RestoreV2:320" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_321" op: "Assign" input: "bert/encoder/layer_4/attention/self/query/bias" input: "save/RestoreV2:321" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_322" op: "Assign" input: "bert/encoder/layer_4/attention/self/query/bias/adam_m" input: "save/RestoreV2:322" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_323" op: "Assign" input: "bert/encoder/layer_4/attention/self/query/bias/adam_v" input: "save/RestoreV2:323" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_324" op: "Assign" input: "bert/encoder/layer_4/attention/self/query/kernel" input: "save/RestoreV2:324" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_325" op: "Assign" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_m" input: "save/RestoreV2:325" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_326" op: "Assign" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_v" input: "save/RestoreV2:326" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_327" op: "Assign" input: "bert/encoder/layer_4/attention/self/value/bias" input: "save/RestoreV2:327" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_328" op: "Assign" input: "bert/encoder/layer_4/attention/self/value/bias/adam_m" input: "save/RestoreV2:328" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_329" op: "Assign" input: "bert/encoder/layer_4/attention/self/value/bias/adam_v" input: "save/RestoreV2:329" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_330" op: "Assign" input: "bert/encoder/layer_4/attention/self/value/kernel" input: "save/RestoreV2:330" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_331" op: "Assign" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_m" input: "save/RestoreV2:331" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_332" op: "Assign" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_v" input: "save/RestoreV2:332" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_333" op: "Assign" input: "bert/encoder/layer_4/intermediate/dense/bias" input: "save/RestoreV2:333" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_334" op: "Assign" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_m" input: "save/RestoreV2:334" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_335" op: "Assign" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_v" input: "save/RestoreV2:335" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_336" op: "Assign" input: "bert/encoder/layer_4/intermediate/dense/kernel" input: "save/RestoreV2:336" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_337" op: "Assign" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m" input: "save/RestoreV2:337" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_338" op: "Assign" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v" input: "save/RestoreV2:338" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_339" op: "Assign" input: "bert/encoder/layer_4/output/LayerNorm/beta" input: "save/RestoreV2:339" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_340" op: "Assign" input: "bert/encoder/layer_4/output/LayerNorm/beta/adam_m" input: "save/RestoreV2:340" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_341" op: "Assign" input: "bert/encoder/layer_4/output/LayerNorm/beta/adam_v" input: "save/RestoreV2:341" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_342" op: "Assign" input: "bert/encoder/layer_4/output/LayerNorm/gamma" input: "save/RestoreV2:342" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_343" op: "Assign" input: "bert/encoder/layer_4/output/LayerNorm/gamma/adam_m" input: "save/RestoreV2:343" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_344" op: "Assign" input: "bert/encoder/layer_4/output/LayerNorm/gamma/adam_v" input: "save/RestoreV2:344" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_345" op: "Assign" input: "bert/encoder/layer_4/output/dense/bias" input: "save/RestoreV2:345" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_346" op: "Assign" input: "bert/encoder/layer_4/output/dense/bias/adam_m" input: "save/RestoreV2:346" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_347" op: "Assign" input: "bert/encoder/layer_4/output/dense/bias/adam_v" input: "save/RestoreV2:347" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_348" op: "Assign" input: "bert/encoder/layer_4/output/dense/kernel" input: "save/RestoreV2:348" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_349" op: "Assign" input: "bert/encoder/layer_4/output/dense/kernel/adam_m" input: "save/RestoreV2:349" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_350" op: "Assign" input: "bert/encoder/layer_4/output/dense/kernel/adam_v" input: "save/RestoreV2:350" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_351" op: "Assign" input: "bert/encoder/layer_5/attention/output/LayerNorm/beta" input: "save/RestoreV2:351" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_352" op: "Assign" input: "bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_m" input: "save/RestoreV2:352" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_353" op: "Assign" input: "bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_v" input: "save/RestoreV2:353" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_354" op: "Assign" input: "bert/encoder/layer_5/attention/output/LayerNorm/gamma" input: "save/RestoreV2:354" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_355" op: "Assign" input: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m" input: "save/RestoreV2:355" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_356" op: "Assign" input: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v" input: "save/RestoreV2:356" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_357" op: "Assign" input: "bert/encoder/layer_5/attention/output/dense/bias" input: "save/RestoreV2:357" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_358" op: "Assign" input: "bert/encoder/layer_5/attention/output/dense/bias/adam_m" input: "save/RestoreV2:358" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_359" op: "Assign" input: "bert/encoder/layer_5/attention/output/dense/bias/adam_v" input: "save/RestoreV2:359" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_360" op: "Assign" input: "bert/encoder/layer_5/attention/output/dense/kernel" input: "save/RestoreV2:360" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_361" op: "Assign" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m" input: "save/RestoreV2:361" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_362" op: "Assign" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v" input: "save/RestoreV2:362" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_363" op: "Assign" input: "bert/encoder/layer_5/attention/self/key/bias" input: "save/RestoreV2:363" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_364" op: "Assign" input: "bert/encoder/layer_5/attention/self/key/bias/adam_m" input: "save/RestoreV2:364" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_365" op: "Assign" input: "bert/encoder/layer_5/attention/self/key/bias/adam_v" input: "save/RestoreV2:365" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_366" op: "Assign" input: "bert/encoder/layer_5/attention/self/key/kernel" input: "save/RestoreV2:366" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_367" op: "Assign" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_m" input: "save/RestoreV2:367" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_368" op: "Assign" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_v" input: "save/RestoreV2:368" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_369" op: "Assign" input: "bert/encoder/layer_5/attention/self/query/bias" input: "save/RestoreV2:369" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_370" op: "Assign" input: "bert/encoder/layer_5/attention/self/query/bias/adam_m" input: "save/RestoreV2:370" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_371" op: "Assign" input: "bert/encoder/layer_5/attention/self/query/bias/adam_v" input: "save/RestoreV2:371" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_372" op: "Assign" input: "bert/encoder/layer_5/attention/self/query/kernel" input: "save/RestoreV2:372" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_373" op: "Assign" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_m" input: "save/RestoreV2:373" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_374" op: "Assign" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_v" input: "save/RestoreV2:374" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_375" op: "Assign" input: "bert/encoder/layer_5/attention/self/value/bias" input: "save/RestoreV2:375" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_376" op: "Assign" input: "bert/encoder/layer_5/attention/self/value/bias/adam_m" input: "save/RestoreV2:376" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_377" op: "Assign" input: "bert/encoder/layer_5/attention/self/value/bias/adam_v" input: "save/RestoreV2:377" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_378" op: "Assign" input: "bert/encoder/layer_5/attention/self/value/kernel" input: "save/RestoreV2:378" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_379" op: "Assign" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_m" input: "save/RestoreV2:379" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_380" op: "Assign" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_v" input: "save/RestoreV2:380" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_381" op: "Assign" input: "bert/encoder/layer_5/intermediate/dense/bias" input: "save/RestoreV2:381" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_382" op: "Assign" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_m" input: "save/RestoreV2:382" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_383" op: "Assign" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_v" input: "save/RestoreV2:383" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_384" op: "Assign" input: "bert/encoder/layer_5/intermediate/dense/kernel" input: "save/RestoreV2:384" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_385" op: "Assign" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m" input: "save/RestoreV2:385" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_386" op: "Assign" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v" input: "save/RestoreV2:386" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_387" op: "Assign" input: "bert/encoder/layer_5/output/LayerNorm/beta" input: "save/RestoreV2:387" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_388" op: "Assign" input: "bert/encoder/layer_5/output/LayerNorm/beta/adam_m" input: "save/RestoreV2:388" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_389" op: "Assign" input: "bert/encoder/layer_5/output/LayerNorm/beta/adam_v" input: "save/RestoreV2:389" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_390" op: "Assign" input: "bert/encoder/layer_5/output/LayerNorm/gamma" input: "save/RestoreV2:390" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_391" op: "Assign" input: "bert/encoder/layer_5/output/LayerNorm/gamma/adam_m" input: "save/RestoreV2:391" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_392" op: "Assign" input: "bert/encoder/layer_5/output/LayerNorm/gamma/adam_v" input: "save/RestoreV2:392" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_393" op: "Assign" input: "bert/encoder/layer_5/output/dense/bias" input: "save/RestoreV2:393" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_394" op: "Assign" input: "bert/encoder/layer_5/output/dense/bias/adam_m" input: "save/RestoreV2:394" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_395" op: "Assign" input: "bert/encoder/layer_5/output/dense/bias/adam_v" input: "save/RestoreV2:395" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_396" op: "Assign" input: "bert/encoder/layer_5/output/dense/kernel" input: "save/RestoreV2:396" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_397" op: "Assign" input: "bert/encoder/layer_5/output/dense/kernel/adam_m" input: "save/RestoreV2:397" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_398" op: "Assign" input: "bert/encoder/layer_5/output/dense/kernel/adam_v" input: "save/RestoreV2:398" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_399" op: "Assign" input: "bert/encoder/layer_6/attention/output/LayerNorm/beta" input: "save/RestoreV2:399" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_400" op: "Assign" input: "bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_m" input: "save/RestoreV2:400" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_401" op: "Assign" input: "bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_v" input: "save/RestoreV2:401" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_402" op: "Assign" input: "bert/encoder/layer_6/attention/output/LayerNorm/gamma" input: "save/RestoreV2:402" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_403" op: "Assign" input: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m" input: "save/RestoreV2:403" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_404" op: "Assign" input: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v" input: "save/RestoreV2:404" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_405" op: "Assign" input: "bert/encoder/layer_6/attention/output/dense/bias" input: "save/RestoreV2:405" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_406" op: "Assign" input: "bert/encoder/layer_6/attention/output/dense/bias/adam_m" input: "save/RestoreV2:406" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_407" op: "Assign" input: "bert/encoder/layer_6/attention/output/dense/bias/adam_v" input: "save/RestoreV2:407" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_408" op: "Assign" input: "bert/encoder/layer_6/attention/output/dense/kernel" input: "save/RestoreV2:408" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_409" op: "Assign" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m" input: "save/RestoreV2:409" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_410" op: "Assign" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v" input: "save/RestoreV2:410" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_411" op: "Assign" input: "bert/encoder/layer_6/attention/self/key/bias" input: "save/RestoreV2:411" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_412" op: "Assign" input: "bert/encoder/layer_6/attention/self/key/bias/adam_m" input: "save/RestoreV2:412" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_413" op: "Assign" input: "bert/encoder/layer_6/attention/self/key/bias/adam_v" input: "save/RestoreV2:413" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_414" op: "Assign" input: "bert/encoder/layer_6/attention/self/key/kernel" input: "save/RestoreV2:414" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_415" op: "Assign" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_m" input: "save/RestoreV2:415" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_416" op: "Assign" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_v" input: "save/RestoreV2:416" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_417" op: "Assign" input: "bert/encoder/layer_6/attention/self/query/bias" input: "save/RestoreV2:417" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_418" op: "Assign" input: "bert/encoder/layer_6/attention/self/query/bias/adam_m" input: "save/RestoreV2:418" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_419" op: "Assign" input: "bert/encoder/layer_6/attention/self/query/bias/adam_v" input: "save/RestoreV2:419" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_420" op: "Assign" input: "bert/encoder/layer_6/attention/self/query/kernel" input: "save/RestoreV2:420" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_421" op: "Assign" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_m" input: "save/RestoreV2:421" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_422" op: "Assign" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_v" input: "save/RestoreV2:422" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_423" op: "Assign" input: "bert/encoder/layer_6/attention/self/value/bias" input: "save/RestoreV2:423" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_424" op: "Assign" input: "bert/encoder/layer_6/attention/self/value/bias/adam_m" input: "save/RestoreV2:424" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_425" op: "Assign" input: "bert/encoder/layer_6/attention/self/value/bias/adam_v" input: "save/RestoreV2:425" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_426" op: "Assign" input: "bert/encoder/layer_6/attention/self/value/kernel" input: "save/RestoreV2:426" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_427" op: "Assign" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_m" input: "save/RestoreV2:427" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_428" op: "Assign" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_v" input: "save/RestoreV2:428" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_429" op: "Assign" input: "bert/encoder/layer_6/intermediate/dense/bias" input: "save/RestoreV2:429" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_430" op: "Assign" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_m" input: "save/RestoreV2:430" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_431" op: "Assign" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_v" input: "save/RestoreV2:431" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_432" op: "Assign" input: "bert/encoder/layer_6/intermediate/dense/kernel" input: "save/RestoreV2:432" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_433" op: "Assign" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m" input: "save/RestoreV2:433" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_434" op: "Assign" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v" input: "save/RestoreV2:434" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_435" op: "Assign" input: "bert/encoder/layer_6/output/LayerNorm/beta" input: "save/RestoreV2:435" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_436" op: "Assign" input: "bert/encoder/layer_6/output/LayerNorm/beta/adam_m" input: "save/RestoreV2:436" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_437" op: "Assign" input: "bert/encoder/layer_6/output/LayerNorm/beta/adam_v" input: "save/RestoreV2:437" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_438" op: "Assign" input: "bert/encoder/layer_6/output/LayerNorm/gamma" input: "save/RestoreV2:438" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_439" op: "Assign" input: "bert/encoder/layer_6/output/LayerNorm/gamma/adam_m" input: "save/RestoreV2:439" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_440" op: "Assign" input: "bert/encoder/layer_6/output/LayerNorm/gamma/adam_v" input: "save/RestoreV2:440" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_441" op: "Assign" input: "bert/encoder/layer_6/output/dense/bias" input: "save/RestoreV2:441" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_442" op: "Assign" input: "bert/encoder/layer_6/output/dense/bias/adam_m" input: "save/RestoreV2:442" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_443" op: "Assign" input: "bert/encoder/layer_6/output/dense/bias/adam_v" input: "save/RestoreV2:443" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_444" op: "Assign" input: "bert/encoder/layer_6/output/dense/kernel" input: "save/RestoreV2:444" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_445" op: "Assign" input: "bert/encoder/layer_6/output/dense/kernel/adam_m" input: "save/RestoreV2:445" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_446" op: "Assign" input: "bert/encoder/layer_6/output/dense/kernel/adam_v" input: "save/RestoreV2:446" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_447" op: "Assign" input: "bert/encoder/layer_7/attention/output/LayerNorm/beta" input: "save/RestoreV2:447" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_448" op: "Assign" input: "bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_m" input: "save/RestoreV2:448" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_449" op: "Assign" input: "bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_v" input: "save/RestoreV2:449" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_450" op: "Assign" input: "bert/encoder/layer_7/attention/output/LayerNorm/gamma" input: "save/RestoreV2:450" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_451" op: "Assign" input: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m" input: "save/RestoreV2:451" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_452" op: "Assign" input: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v" input: "save/RestoreV2:452" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_453" op: "Assign" input: "bert/encoder/layer_7/attention/output/dense/bias" input: "save/RestoreV2:453" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_454" op: "Assign" input: "bert/encoder/layer_7/attention/output/dense/bias/adam_m" input: "save/RestoreV2:454" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_455" op: "Assign" input: "bert/encoder/layer_7/attention/output/dense/bias/adam_v" input: "save/RestoreV2:455" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_456" op: "Assign" input: "bert/encoder/layer_7/attention/output/dense/kernel" input: "save/RestoreV2:456" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_457" op: "Assign" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m" input: "save/RestoreV2:457" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_458" op: "Assign" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v" input: "save/RestoreV2:458" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_459" op: "Assign" input: "bert/encoder/layer_7/attention/self/key/bias" input: "save/RestoreV2:459" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_460" op: "Assign" input: "bert/encoder/layer_7/attention/self/key/bias/adam_m" input: "save/RestoreV2:460" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_461" op: "Assign" input: "bert/encoder/layer_7/attention/self/key/bias/adam_v" input: "save/RestoreV2:461" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_462" op: "Assign" input: "bert/encoder/layer_7/attention/self/key/kernel" input: "save/RestoreV2:462" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_463" op: "Assign" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_m" input: "save/RestoreV2:463" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_464" op: "Assign" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_v" input: "save/RestoreV2:464" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_465" op: "Assign" input: "bert/encoder/layer_7/attention/self/query/bias" input: "save/RestoreV2:465" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_466" op: "Assign" input: "bert/encoder/layer_7/attention/self/query/bias/adam_m" input: "save/RestoreV2:466" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_467" op: "Assign" input: "bert/encoder/layer_7/attention/self/query/bias/adam_v" input: "save/RestoreV2:467" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_468" op: "Assign" input: "bert/encoder/layer_7/attention/self/query/kernel" input: "save/RestoreV2:468" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_469" op: "Assign" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_m" input: "save/RestoreV2:469" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_470" op: "Assign" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_v" input: "save/RestoreV2:470" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_471" op: "Assign" input: "bert/encoder/layer_7/attention/self/value/bias" input: "save/RestoreV2:471" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_472" op: "Assign" input: "bert/encoder/layer_7/attention/self/value/bias/adam_m" input: "save/RestoreV2:472" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_473" op: "Assign" input: "bert/encoder/layer_7/attention/self/value/bias/adam_v" input: "save/RestoreV2:473" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_474" op: "Assign" input: "bert/encoder/layer_7/attention/self/value/kernel" input: "save/RestoreV2:474" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_475" op: "Assign" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_m" input: "save/RestoreV2:475" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_476" op: "Assign" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_v" input: "save/RestoreV2:476" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_477" op: "Assign" input: "bert/encoder/layer_7/intermediate/dense/bias" input: "save/RestoreV2:477" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_478" op: "Assign" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_m" input: "save/RestoreV2:478" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_479" op: "Assign" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_v" input: "save/RestoreV2:479" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_480" op: "Assign" input: "bert/encoder/layer_7/intermediate/dense/kernel" input: "save/RestoreV2:480" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_481" op: "Assign" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m" input: "save/RestoreV2:481" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_482" op: "Assign" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v" input: "save/RestoreV2:482" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_483" op: "Assign" input: "bert/encoder/layer_7/output/LayerNorm/beta" input: "save/RestoreV2:483" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_484" op: "Assign" input: "bert/encoder/layer_7/output/LayerNorm/beta/adam_m" input: "save/RestoreV2:484" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_485" op: "Assign" input: "bert/encoder/layer_7/output/LayerNorm/beta/adam_v" input: "save/RestoreV2:485" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_486" op: "Assign" input: "bert/encoder/layer_7/output/LayerNorm/gamma" input: "save/RestoreV2:486" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_487" op: "Assign" input: "bert/encoder/layer_7/output/LayerNorm/gamma/adam_m" input: "save/RestoreV2:487" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_488" op: "Assign" input: "bert/encoder/layer_7/output/LayerNorm/gamma/adam_v" input: "save/RestoreV2:488" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_489" op: "Assign" input: "bert/encoder/layer_7/output/dense/bias" input: "save/RestoreV2:489" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_490" op: "Assign" input: "bert/encoder/layer_7/output/dense/bias/adam_m" input: "save/RestoreV2:490" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_491" op: "Assign" input: "bert/encoder/layer_7/output/dense/bias/adam_v" input: "save/RestoreV2:491" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_492" op: "Assign" input: "bert/encoder/layer_7/output/dense/kernel" input: "save/RestoreV2:492" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_493" op: "Assign" input: "bert/encoder/layer_7/output/dense/kernel/adam_m" input: "save/RestoreV2:493" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_494" op: "Assign" input: "bert/encoder/layer_7/output/dense/kernel/adam_v" input: "save/RestoreV2:494" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_495" op: "Assign" input: "bert/encoder/layer_8/attention/output/LayerNorm/beta" input: "save/RestoreV2:495" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_496" op: "Assign" input: "bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_m" input: "save/RestoreV2:496" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_497" op: "Assign" input: "bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_v" input: "save/RestoreV2:497" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_498" op: "Assign" input: "bert/encoder/layer_8/attention/output/LayerNorm/gamma" input: "save/RestoreV2:498" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_499" op: "Assign" input: "bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m" input: "save/RestoreV2:499" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_500" op: "Assign" input: "bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v" input: "save/RestoreV2:500" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_501" op: "Assign" input: "bert/encoder/layer_8/attention/output/dense/bias" input: "save/RestoreV2:501" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_502" op: "Assign" input: "bert/encoder/layer_8/attention/output/dense/bias/adam_m" input: "save/RestoreV2:502" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_503" op: "Assign" input: "bert/encoder/layer_8/attention/output/dense/bias/adam_v" input: "save/RestoreV2:503" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_504" op: "Assign" input: "bert/encoder/layer_8/attention/output/dense/kernel" input: "save/RestoreV2:504" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_505" op: "Assign" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m" input: "save/RestoreV2:505" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_506" op: "Assign" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v" input: "save/RestoreV2:506" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_507" op: "Assign" input: "bert/encoder/layer_8/attention/self/key/bias" input: "save/RestoreV2:507" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_508" op: "Assign" input: "bert/encoder/layer_8/attention/self/key/bias/adam_m" input: "save/RestoreV2:508" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_509" op: "Assign" input: "bert/encoder/layer_8/attention/self/key/bias/adam_v" input: "save/RestoreV2:509" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_510" op: "Assign" input: "bert/encoder/layer_8/attention/self/key/kernel" input: "save/RestoreV2:510" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_511" op: "Assign" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_m" input: "save/RestoreV2:511" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_512" op: "Assign" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_v" input: "save/RestoreV2:512" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_513" op: "Assign" input: "bert/encoder/layer_8/attention/self/query/bias" input: "save/RestoreV2:513" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_514" op: "Assign" input: "bert/encoder/layer_8/attention/self/query/bias/adam_m" input: "save/RestoreV2:514" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_515" op: "Assign" input: "bert/encoder/layer_8/attention/self/query/bias/adam_v" input: "save/RestoreV2:515" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_516" op: "Assign" input: "bert/encoder/layer_8/attention/self/query/kernel" input: "save/RestoreV2:516" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_517" op: "Assign" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_m" input: "save/RestoreV2:517" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_518" op: "Assign" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_v" input: "save/RestoreV2:518" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_519" op: "Assign" input: "bert/encoder/layer_8/attention/self/value/bias" input: "save/RestoreV2:519" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_520" op: "Assign" input: "bert/encoder/layer_8/attention/self/value/bias/adam_m" input: "save/RestoreV2:520" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_521" op: "Assign" input: "bert/encoder/layer_8/attention/self/value/bias/adam_v" input: "save/RestoreV2:521" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_522" op: "Assign" input: "bert/encoder/layer_8/attention/self/value/kernel" input: "save/RestoreV2:522" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_523" op: "Assign" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_m" input: "save/RestoreV2:523" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_524" op: "Assign" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_v" input: "save/RestoreV2:524" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_525" op: "Assign" input: "bert/encoder/layer_8/intermediate/dense/bias" input: "save/RestoreV2:525" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_526" op: "Assign" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_m" input: "save/RestoreV2:526" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_527" op: "Assign" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_v" input: "save/RestoreV2:527" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_528" op: "Assign" input: "bert/encoder/layer_8/intermediate/dense/kernel" input: "save/RestoreV2:528" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_529" op: "Assign" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m" input: "save/RestoreV2:529" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_530" op: "Assign" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v" input: "save/RestoreV2:530" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_531" op: "Assign" input: "bert/encoder/layer_8/output/LayerNorm/beta" input: "save/RestoreV2:531" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_532" op: "Assign" input: "bert/encoder/layer_8/output/LayerNorm/beta/adam_m" input: "save/RestoreV2:532" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_533" op: "Assign" input: "bert/encoder/layer_8/output/LayerNorm/beta/adam_v" input: "save/RestoreV2:533" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_534" op: "Assign" input: "bert/encoder/layer_8/output/LayerNorm/gamma" input: "save/RestoreV2:534" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_535" op: "Assign" input: "bert/encoder/layer_8/output/LayerNorm/gamma/adam_m" input: "save/RestoreV2:535" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_536" op: "Assign" input: "bert/encoder/layer_8/output/LayerNorm/gamma/adam_v" input: "save/RestoreV2:536" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_537" op: "Assign" input: "bert/encoder/layer_8/output/dense/bias" input: "save/RestoreV2:537" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_538" op: "Assign" input: "bert/encoder/layer_8/output/dense/bias/adam_m" input: "save/RestoreV2:538" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_539" op: "Assign" input: "bert/encoder/layer_8/output/dense/bias/adam_v" input: "save/RestoreV2:539" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_540" op: "Assign" input: "bert/encoder/layer_8/output/dense/kernel" input: "save/RestoreV2:540" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_541" op: "Assign" input: "bert/encoder/layer_8/output/dense/kernel/adam_m" input: "save/RestoreV2:541" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_542" op: "Assign" input: "bert/encoder/layer_8/output/dense/kernel/adam_v" input: "save/RestoreV2:542" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_543" op: "Assign" input: "bert/encoder/layer_9/attention/output/LayerNorm/beta" input: "save/RestoreV2:543" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_544" op: "Assign" input: "bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_m" input: "save/RestoreV2:544" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_545" op: "Assign" input: "bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_v" input: "save/RestoreV2:545" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_546" op: "Assign" input: "bert/encoder/layer_9/attention/output/LayerNorm/gamma" input: "save/RestoreV2:546" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_547" op: "Assign" input: "bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m" input: "save/RestoreV2:547" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_548" op: "Assign" input: "bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v" input: "save/RestoreV2:548" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_549" op: "Assign" input: "bert/encoder/layer_9/attention/output/dense/bias" input: "save/RestoreV2:549" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_550" op: "Assign" input: "bert/encoder/layer_9/attention/output/dense/bias/adam_m" input: "save/RestoreV2:550" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_551" op: "Assign" input: "bert/encoder/layer_9/attention/output/dense/bias/adam_v" input: "save/RestoreV2:551" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_552" op: "Assign" input: "bert/encoder/layer_9/attention/output/dense/kernel" input: "save/RestoreV2:552" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_553" op: "Assign" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m" input: "save/RestoreV2:553" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_554" op: "Assign" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v" input: "save/RestoreV2:554" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_555" op: "Assign" input: "bert/encoder/layer_9/attention/self/key/bias" input: "save/RestoreV2:555" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_556" op: "Assign" input: "bert/encoder/layer_9/attention/self/key/bias/adam_m" input: "save/RestoreV2:556" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_557" op: "Assign" input: "bert/encoder/layer_9/attention/self/key/bias/adam_v" input: "save/RestoreV2:557" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_558" op: "Assign" input: "bert/encoder/layer_9/attention/self/key/kernel" input: "save/RestoreV2:558" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_559" op: "Assign" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_m" input: "save/RestoreV2:559" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_560" op: "Assign" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_v" input: "save/RestoreV2:560" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_561" op: "Assign" input: "bert/encoder/layer_9/attention/self/query/bias" input: "save/RestoreV2:561" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_562" op: "Assign" input: "bert/encoder/layer_9/attention/self/query/bias/adam_m" input: "save/RestoreV2:562" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_563" op: "Assign" input: "bert/encoder/layer_9/attention/self/query/bias/adam_v" input: "save/RestoreV2:563" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_564" op: "Assign" input: "bert/encoder/layer_9/attention/self/query/kernel" input: "save/RestoreV2:564" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_565" op: "Assign" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_m" input: "save/RestoreV2:565" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_566" op: "Assign" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_v" input: "save/RestoreV2:566" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_567" op: "Assign" input: "bert/encoder/layer_9/attention/self/value/bias" input: "save/RestoreV2:567" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_568" op: "Assign" input: "bert/encoder/layer_9/attention/self/value/bias/adam_m" input: "save/RestoreV2:568" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_569" op: "Assign" input: "bert/encoder/layer_9/attention/self/value/bias/adam_v" input: "save/RestoreV2:569" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_570" op: "Assign" input: "bert/encoder/layer_9/attention/self/value/kernel" input: "save/RestoreV2:570" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_571" op: "Assign" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_m" input: "save/RestoreV2:571" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_572" op: "Assign" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_v" input: "save/RestoreV2:572" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_573" op: "Assign" input: "bert/encoder/layer_9/intermediate/dense/bias" input: "save/RestoreV2:573" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_574" op: "Assign" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_m" input: "save/RestoreV2:574" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_575" op: "Assign" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_v" input: "save/RestoreV2:575" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_576" op: "Assign" input: "bert/encoder/layer_9/intermediate/dense/kernel" input: "save/RestoreV2:576" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_577" op: "Assign" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m" input: "save/RestoreV2:577" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_578" op: "Assign" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v" input: "save/RestoreV2:578" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_579" op: "Assign" input: "bert/encoder/layer_9/output/LayerNorm/beta" input: "save/RestoreV2:579" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_580" op: "Assign" input: "bert/encoder/layer_9/output/LayerNorm/beta/adam_m" input: "save/RestoreV2:580" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_581" op: "Assign" input: "bert/encoder/layer_9/output/LayerNorm/beta/adam_v" input: "save/RestoreV2:581" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_582" op: "Assign" input: "bert/encoder/layer_9/output/LayerNorm/gamma" input: "save/RestoreV2:582" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_583" op: "Assign" input: "bert/encoder/layer_9/output/LayerNorm/gamma/adam_m" input: "save/RestoreV2:583" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_584" op: "Assign" input: "bert/encoder/layer_9/output/LayerNorm/gamma/adam_v" input: "save/RestoreV2:584" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_585" op: "Assign" input: "bert/encoder/layer_9/output/dense/bias" input: "save/RestoreV2:585" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_586" op: "Assign" input: "bert/encoder/layer_9/output/dense/bias/adam_m" input: "save/RestoreV2:586" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_587" op: "Assign" input: "bert/encoder/layer_9/output/dense/bias/adam_v" input: "save/RestoreV2:587" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_588" op: "Assign" input: "bert/encoder/layer_9/output/dense/kernel" input: "save/RestoreV2:588" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_589" op: "Assign" input: "bert/encoder/layer_9/output/dense/kernel/adam_m" input: "save/RestoreV2:589" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_590" op: "Assign" input: "bert/encoder/layer_9/output/dense/kernel/adam_v" input: "save/RestoreV2:590" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_591" op: "Assign" input: "bert/pooler/dense/bias" input: "save/RestoreV2:591" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_592" op: "Assign" input: "bert/pooler/dense/bias/adam_m" input: "save/RestoreV2:592" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_593" op: "Assign" input: "bert/pooler/dense/bias/adam_v" input: "save/RestoreV2:593" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_594" op: "Assign" input: "bert/pooler/dense/kernel" input: "save/RestoreV2:594" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_595" op: "Assign" input: "bert/pooler/dense/kernel/adam_m" input: "save/RestoreV2:595" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_596" op: "Assign" input: "bert/pooler/dense/kernel/adam_v" input: "save/RestoreV2:596" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_597" op: "Assign" input: "cls/predictions/output_bias" input: "save/RestoreV2:597" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/output_bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_598" op: "Assign" input: "cls/predictions/output_bias/adam_m" input: "save/RestoreV2:598" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/output_bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_599" op: "Assign" input: "cls/predictions/output_bias/adam_v" input: "save/RestoreV2:599" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/output_bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 28996 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_600" op: "Assign" input: "cls/predictions/transform/LayerNorm/beta" input: "save/RestoreV2:600" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_601" op: "Assign" input: "cls/predictions/transform/LayerNorm/beta/adam_m" input: "save/RestoreV2:601" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_602" op: "Assign" input: "cls/predictions/transform/LayerNorm/beta/adam_v" input: "save/RestoreV2:602" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_603" op: "Assign" input: "cls/predictions/transform/LayerNorm/gamma" input: "save/RestoreV2:603" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_604" op: "Assign" input: "cls/predictions/transform/LayerNorm/gamma/adam_m" input: "save/RestoreV2:604" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_605" op: "Assign" input: "cls/predictions/transform/LayerNorm/gamma/adam_v" input: "save/RestoreV2:605" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/LayerNorm/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_606" op: "Assign" input: "cls/predictions/transform/dense/bias" input: "save/RestoreV2:606" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_607" op: "Assign" input: "cls/predictions/transform/dense/bias/adam_m" input: "save/RestoreV2:607" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_608" op: "Assign" input: "cls/predictions/transform/dense/bias/adam_v" input: "save/RestoreV2:608" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_609" op: "Assign" input: "cls/predictions/transform/dense/kernel" input: "save/RestoreV2:609" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_610" op: "Assign" input: "cls/predictions/transform/dense/kernel/adam_m" input: "save/RestoreV2:610" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_611" op: "Assign" input: "cls/predictions/transform/dense/kernel/adam_v" input: "save/RestoreV2:611" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/predictions/transform/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_612" op: "Assign" input: "cls/seq_relationship/output_bias" input: "save/RestoreV2:612" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_613" op: "Assign" input: "cls/seq_relationship/output_bias/adam_m" input: "save/RestoreV2:613" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_614" op: "Assign" input: "cls/seq_relationship/output_bias/adam_v" input: "save/RestoreV2:614" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_615" op: "Assign" input: "cls/seq_relationship/output_weights" input: "save/RestoreV2:615" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_weights" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_616" op: "Assign" input: "cls/seq_relationship/output_weights/adam_m" input: "save/RestoreV2:616" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_weights/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Assign_617" op: "Assign" input: "cls/seq_relationship/output_weights/adam_v" input: "save/RestoreV2:617" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@cls/seq_relationship/output_weights/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "use_locking" value { b: true } } attr { key: "validate_shape" value { b: true } } } node { name: "save/Identity_1" op: "Identity" input: "save/RestoreV2:618" attr { key: "T" value { type: DT_INT64 } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp" op: "AssignVariableOp" input: "global_step" input: "save/Identity_1" attr { key: "dtype" value { type: DT_INT64 } } } node { name: "save/restore_shard" op: "NoOp" input: "^save/Assign" input: "^save/AssignVariableOp" input: "^save/Assign_1" input: "^save/Assign_10" input: "^save/Assign_100" input: "^save/Assign_101" input: "^save/Assign_102" input: "^save/Assign_103" input: "^save/Assign_104" input: "^save/Assign_105" input: "^save/Assign_106" input: "^save/Assign_107" input: "^save/Assign_108" input: "^save/Assign_109" input: "^save/Assign_11" input: "^save/Assign_110" input: "^save/Assign_111" input: "^save/Assign_112" input: "^save/Assign_113" input: "^save/Assign_114" input: "^save/Assign_115" input: "^save/Assign_116" input: "^save/Assign_117" input: "^save/Assign_118" input: "^save/Assign_119" input: "^save/Assign_12" input: "^save/Assign_120" input: "^save/Assign_121" input: "^save/Assign_122" input: "^save/Assign_123" input: "^save/Assign_124" input: "^save/Assign_125" input: "^save/Assign_126" input: "^save/Assign_127" input: "^save/Assign_128" input: "^save/Assign_129" input: "^save/Assign_13" input: "^save/Assign_130" input: "^save/Assign_131" input: "^save/Assign_132" input: "^save/Assign_133" input: "^save/Assign_134" input: "^save/Assign_135" input: "^save/Assign_136" input: "^save/Assign_137" input: "^save/Assign_138" input: "^save/Assign_139" input: "^save/Assign_14" input: "^save/Assign_140" input: "^save/Assign_141" input: "^save/Assign_142" input: "^save/Assign_143" input: "^save/Assign_144" input: "^save/Assign_145" input: "^save/Assign_146" input: "^save/Assign_147" input: "^save/Assign_148" input: "^save/Assign_149" input: "^save/Assign_15" input: "^save/Assign_150" input: "^save/Assign_151" input: "^save/Assign_152" input: "^save/Assign_153" input: "^save/Assign_154" input: "^save/Assign_155" input: "^save/Assign_156" input: "^save/Assign_157" input: "^save/Assign_158" input: "^save/Assign_159" input: "^save/Assign_16" input: "^save/Assign_160" input: "^save/Assign_161" input: "^save/Assign_162" input: "^save/Assign_163" input: "^save/Assign_164" input: "^save/Assign_165" input: "^save/Assign_166" input: "^save/Assign_167" input: "^save/Assign_168" input: "^save/Assign_169" input: "^save/Assign_17" input: "^save/Assign_170" input: "^save/Assign_171" input: "^save/Assign_172" input: "^save/Assign_173" input: "^save/Assign_174" input: "^save/Assign_175" input: "^save/Assign_176" input: "^save/Assign_177" input: "^save/Assign_178" input: "^save/Assign_179" input: "^save/Assign_18" input: "^save/Assign_180" input: "^save/Assign_181" input: "^save/Assign_182" input: "^save/Assign_183" input: "^save/Assign_184" input: "^save/Assign_185" input: "^save/Assign_186" input: "^save/Assign_187" input: "^save/Assign_188" input: "^save/Assign_189" input: "^save/Assign_19" input: "^save/Assign_190" input: "^save/Assign_191" input: "^save/Assign_192" input: "^save/Assign_193" input: "^save/Assign_194" input: "^save/Assign_195" input: "^save/Assign_196" input: "^save/Assign_197" input: "^save/Assign_198" input: "^save/Assign_199" input: "^save/Assign_2" input: "^save/Assign_20" input: "^save/Assign_200" input: "^save/Assign_201" input: "^save/Assign_202" input: "^save/Assign_203" input: "^save/Assign_204" input: "^save/Assign_205" input: "^save/Assign_206" input: "^save/Assign_207" input: "^save/Assign_208" input: "^save/Assign_209" input: "^save/Assign_21" input: "^save/Assign_210" input: "^save/Assign_211" input: "^save/Assign_212" input: "^save/Assign_213" input: "^save/Assign_214" input: "^save/Assign_215" input: "^save/Assign_216" input: "^save/Assign_217" input: "^save/Assign_218" input: "^save/Assign_219" input: "^save/Assign_22" input: "^save/Assign_220" input: "^save/Assign_221" input: "^save/Assign_222" input: "^save/Assign_223" input: "^save/Assign_224" input: "^save/Assign_225" input: "^save/Assign_226" input: "^save/Assign_227" input: "^save/Assign_228" input: "^save/Assign_229" input: "^save/Assign_23" input: "^save/Assign_230" input: "^save/Assign_231" input: "^save/Assign_232" input: "^save/Assign_233" input: "^save/Assign_234" input: "^save/Assign_235" input: "^save/Assign_236" input: "^save/Assign_237" input: "^save/Assign_238" input: "^save/Assign_239" input: "^save/Assign_24" input: "^save/Assign_240" input: "^save/Assign_241" input: "^save/Assign_242" input: "^save/Assign_243" input: "^save/Assign_244" input: "^save/Assign_245" input: "^save/Assign_246" input: "^save/Assign_247" input: "^save/Assign_248" input: "^save/Assign_249" input: "^save/Assign_25" input: "^save/Assign_250" input: "^save/Assign_251" input: "^save/Assign_252" input: "^save/Assign_253" input: "^save/Assign_254" input: "^save/Assign_255" input: "^save/Assign_256" input: "^save/Assign_257" input: "^save/Assign_258" input: "^save/Assign_259" input: "^save/Assign_26" input: "^save/Assign_260" input: "^save/Assign_261" input: "^save/Assign_262" input: "^save/Assign_263" input: "^save/Assign_264" input: "^save/Assign_265" input: "^save/Assign_266" input: "^save/Assign_267" input: "^save/Assign_268" input: "^save/Assign_269" input: "^save/Assign_27" input: "^save/Assign_270" input: "^save/Assign_271" input: "^save/Assign_272" input: "^save/Assign_273" input: "^save/Assign_274" input: "^save/Assign_275" input: "^save/Assign_276" input: "^save/Assign_277" input: "^save/Assign_278" input: "^save/Assign_279" input: "^save/Assign_28" input: "^save/Assign_280" input: "^save/Assign_281" input: "^save/Assign_282" input: "^save/Assign_283" input: "^save/Assign_284" input: "^save/Assign_285" input: "^save/Assign_286" input: "^save/Assign_287" input: "^save/Assign_288" input: "^save/Assign_289" input: "^save/Assign_29" input: "^save/Assign_290" input: "^save/Assign_291" input: "^save/Assign_292" input: "^save/Assign_293" input: "^save/Assign_294" input: "^save/Assign_295" input: "^save/Assign_296" input: "^save/Assign_297" input: "^save/Assign_298" input: "^save/Assign_299" input: "^save/Assign_3" input: "^save/Assign_30" input: "^save/Assign_300" input: "^save/Assign_301" input: "^save/Assign_302" input: "^save/Assign_303" input: "^save/Assign_304" input: "^save/Assign_305" input: "^save/Assign_306" input: "^save/Assign_307" input: "^save/Assign_308" input: "^save/Assign_309" input: "^save/Assign_31" input: "^save/Assign_310" input: "^save/Assign_311" input: "^save/Assign_312" input: "^save/Assign_313" input: "^save/Assign_314" input: "^save/Assign_315" input: "^save/Assign_316" input: "^save/Assign_317" input: "^save/Assign_318" input: "^save/Assign_319" input: "^save/Assign_32" input: "^save/Assign_320" input: "^save/Assign_321" input: "^save/Assign_322" input: "^save/Assign_323" input: "^save/Assign_324" input: "^save/Assign_325" input: "^save/Assign_326" input: "^save/Assign_327" input: "^save/Assign_328" input: "^save/Assign_329" input: "^save/Assign_33" input: "^save/Assign_330" input: "^save/Assign_331" input: "^save/Assign_332" input: "^save/Assign_333" input: "^save/Assign_334" input: "^save/Assign_335" input: "^save/Assign_336" input: "^save/Assign_337" input: "^save/Assign_338" input: "^save/Assign_339" input: "^save/Assign_34" input: "^save/Assign_340" input: "^save/Assign_341" input: "^save/Assign_342" input: "^save/Assign_343" input: "^save/Assign_344" input: "^save/Assign_345" input: "^save/Assign_346" input: "^save/Assign_347" input: "^save/Assign_348" input: "^save/Assign_349" input: "^save/Assign_35" input: "^save/Assign_350" input: "^save/Assign_351" input: "^save/Assign_352" input: "^save/Assign_353" input: "^save/Assign_354" input: "^save/Assign_355" input: "^save/Assign_356" input: "^save/Assign_357" input: "^save/Assign_358" input: "^save/Assign_359" input: "^save/Assign_36" input: "^save/Assign_360" input: "^save/Assign_361" input: "^save/Assign_362" input: "^save/Assign_363" input: "^save/Assign_364" input: "^save/Assign_365" input: "^save/Assign_366" input: "^save/Assign_367" input: "^save/Assign_368" input: "^save/Assign_369" input: "^save/Assign_37" input: "^save/Assign_370" input: "^save/Assign_371" input: "^save/Assign_372" input: "^save/Assign_373" input: "^save/Assign_374" input: "^save/Assign_375" input: "^save/Assign_376" input: "^save/Assign_377" input: "^save/Assign_378" input: "^save/Assign_379" input: "^save/Assign_38" input: "^save/Assign_380" input: "^save/Assign_381" input: "^save/Assign_382" input: "^save/Assign_383" input: "^save/Assign_384" input: "^save/Assign_385" input: "^save/Assign_386" input: "^save/Assign_387" input: "^save/Assign_388" input: "^save/Assign_389" input: "^save/Assign_39" input: "^save/Assign_390" input: "^save/Assign_391" input: "^save/Assign_392" input: "^save/Assign_393" input: "^save/Assign_394" input: "^save/Assign_395" input: "^save/Assign_396" input: "^save/Assign_397" input: "^save/Assign_398" input: "^save/Assign_399" input: "^save/Assign_4" input: "^save/Assign_40" input: "^save/Assign_400" input: "^save/Assign_401" input: "^save/Assign_402" input: "^save/Assign_403" input: "^save/Assign_404" input: "^save/Assign_405" input: "^save/Assign_406" input: "^save/Assign_407" input: "^save/Assign_408" input: "^save/Assign_409" input: "^save/Assign_41" input: "^save/Assign_410" input: "^save/Assign_411" input: "^save/Assign_412" input: "^save/Assign_413" input: "^save/Assign_414" input: "^save/Assign_415" input: "^save/Assign_416" input: "^save/Assign_417" input: "^save/Assign_418" input: "^save/Assign_419" input: "^save/Assign_42" input: "^save/Assign_420" input: "^save/Assign_421" input: "^save/Assign_422" input: "^save/Assign_423" input: "^save/Assign_424" input: "^save/Assign_425" input: "^save/Assign_426" input: "^save/Assign_427" input: "^save/Assign_428" input: "^save/Assign_429" input: "^save/Assign_43" input: "^save/Assign_430" input: "^save/Assign_431" input: "^save/Assign_432" input: "^save/Assign_433" input: "^save/Assign_434" input: "^save/Assign_435" input: "^save/Assign_436" input: "^save/Assign_437" input: "^save/Assign_438" input: "^save/Assign_439" input: "^save/Assign_44" input: "^save/Assign_440" input: "^save/Assign_441" input: "^save/Assign_442" input: "^save/Assign_443" input: "^save/Assign_444" input: "^save/Assign_445" input: "^save/Assign_446" input: "^save/Assign_447" input: "^save/Assign_448" input: "^save/Assign_449" input: "^save/Assign_45" input: "^save/Assign_450" input: "^save/Assign_451" input: "^save/Assign_452" input: "^save/Assign_453" input: "^save/Assign_454" input: "^save/Assign_455" input: "^save/Assign_456" input: "^save/Assign_457" input: "^save/Assign_458" input: "^save/Assign_459" input: "^save/Assign_46" input: "^save/Assign_460" input: "^save/Assign_461" input: "^save/Assign_462" input: "^save/Assign_463" input: "^save/Assign_464" input: "^save/Assign_465" input: "^save/Assign_466" input: "^save/Assign_467" input: "^save/Assign_468" input: "^save/Assign_469" input: "^save/Assign_47" input: "^save/Assign_470" input: "^save/Assign_471" input: "^save/Assign_472" input: "^save/Assign_473" input: "^save/Assign_474" input: "^save/Assign_475" input: "^save/Assign_476" input: "^save/Assign_477" input: "^save/Assign_478" input: "^save/Assign_479" input: "^save/Assign_48" input: "^save/Assign_480" input: "^save/Assign_481" input: "^save/Assign_482" input: "^save/Assign_483" input: "^save/Assign_484" input: "^save/Assign_485" input: "^save/Assign_486" input: "^save/Assign_487" input: "^save/Assign_488" input: "^save/Assign_489" input: "^save/Assign_49" input: "^save/Assign_490" input: "^save/Assign_491" input: "^save/Assign_492" input: "^save/Assign_493" input: "^save/Assign_494" input: "^save/Assign_495" input: "^save/Assign_496" input: "^save/Assign_497" input: "^save/Assign_498" input: "^save/Assign_499" input: "^save/Assign_5" input: "^save/Assign_50" input: "^save/Assign_500" input: "^save/Assign_501" input: "^save/Assign_502" input: "^save/Assign_503" input: "^save/Assign_504" input: "^save/Assign_505" input: "^save/Assign_506" input: "^save/Assign_507" input: "^save/Assign_508" input: "^save/Assign_509" input: "^save/Assign_51" input: "^save/Assign_510" input: "^save/Assign_511" input: "^save/Assign_512" input: "^save/Assign_513" input: "^save/Assign_514" input: "^save/Assign_515" input: "^save/Assign_516" input: "^save/Assign_517" input: "^save/Assign_518" input: "^save/Assign_519" input: "^save/Assign_52" input: "^save/Assign_520" input: "^save/Assign_521" input: "^save/Assign_522" input: "^save/Assign_523" input: "^save/Assign_524" input: "^save/Assign_525" input: "^save/Assign_526" input: "^save/Assign_527" input: "^save/Assign_528" input: "^save/Assign_529" input: "^save/Assign_53" input: "^save/Assign_530" input: "^save/Assign_531" input: "^save/Assign_532" input: "^save/Assign_533" input: "^save/Assign_534" input: "^save/Assign_535" input: "^save/Assign_536" input: "^save/Assign_537" input: "^save/Assign_538" input: "^save/Assign_539" input: "^save/Assign_54" input: "^save/Assign_540" input: "^save/Assign_541" input: "^save/Assign_542" input: "^save/Assign_543" input: "^save/Assign_544" input: "^save/Assign_545" input: "^save/Assign_546" input: "^save/Assign_547" input: "^save/Assign_548" input: "^save/Assign_549" input: "^save/Assign_55" input: "^save/Assign_550" input: "^save/Assign_551" input: "^save/Assign_552" input: "^save/Assign_553" input: "^save/Assign_554" input: "^save/Assign_555" input: "^save/Assign_556" input: "^save/Assign_557" input: "^save/Assign_558" input: "^save/Assign_559" input: "^save/Assign_56" input: "^save/Assign_560" input: "^save/Assign_561" input: "^save/Assign_562" input: "^save/Assign_563" input: "^save/Assign_564" input: "^save/Assign_565" input: "^save/Assign_566" input: "^save/Assign_567" input: "^save/Assign_568" input: "^save/Assign_569" input: "^save/Assign_57" input: "^save/Assign_570" input: "^save/Assign_571" input: "^save/Assign_572" input: "^save/Assign_573" input: "^save/Assign_574" input: "^save/Assign_575" input: "^save/Assign_576" input: "^save/Assign_577" input: "^save/Assign_578" input: "^save/Assign_579" input: "^save/Assign_58" input: "^save/Assign_580" input: "^save/Assign_581" input: "^save/Assign_582" input: "^save/Assign_583" input: "^save/Assign_584" input: "^save/Assign_585" input: "^save/Assign_586" input: "^save/Assign_587" input: "^save/Assign_588" input: "^save/Assign_589" input: "^save/Assign_59" input: "^save/Assign_590" input: "^save/Assign_591" input: "^save/Assign_592" input: "^save/Assign_593" input: "^save/Assign_594" input: "^save/Assign_595" input: "^save/Assign_596" input: "^save/Assign_597" input: "^save/Assign_598" input: "^save/Assign_599" input: "^save/Assign_6" input: "^save/Assign_60" input: "^save/Assign_600" input: "^save/Assign_601" input: "^save/Assign_602" input: "^save/Assign_603" input: "^save/Assign_604" input: "^save/Assign_605" input: "^save/Assign_606" input: "^save/Assign_607" input: "^save/Assign_608" input: "^save/Assign_609" input: "^save/Assign_61" input: "^save/Assign_610" input: "^save/Assign_611" input: "^save/Assign_612" input: "^save/Assign_613" input: "^save/Assign_614" input: "^save/Assign_615" input: "^save/Assign_616" input: "^save/Assign_617" input: "^save/Assign_62" input: "^save/Assign_63" input: "^save/Assign_64" input: "^save/Assign_65" input: "^save/Assign_66" input: "^save/Assign_67" input: "^save/Assign_68" input: "^save/Assign_69" input: "^save/Assign_7" input: "^save/Assign_70" input: "^save/Assign_71" input: "^save/Assign_72" input: "^save/Assign_73" input: "^save/Assign_74" input: "^save/Assign_75" input: "^save/Assign_76" input: "^save/Assign_77" input: "^save/Assign_78" input: "^save/Assign_79" input: "^save/Assign_8" input: "^save/Assign_80" input: "^save/Assign_81" input: "^save/Assign_82" input: "^save/Assign_83" input: "^save/Assign_84" input: "^save/Assign_85" input: "^save/Assign_86" input: "^save/Assign_87" input: "^save/Assign_88" input: "^save/Assign_89" input: "^save/Assign_9" input: "^save/Assign_90" input: "^save/Assign_91" input: "^save/Assign_92" input: "^save/Assign_93" input: "^save/Assign_94" input: "^save/Assign_95" input: "^save/Assign_96" input: "^save/Assign_97" input: "^save/Assign_98" input: "^save/Assign_99" } node { name: "save/restore_all" op: "NoOp" input: "^save/restore_shard" } library { function { signature { name: "tf_data_structured_function_wrapper_9z4XDFOWUdQ" input_arg { name: "arg0" type: DT_STRING } output_arg { name: "toint32" type: DT_INT32 } output_arg { name: "toint32_1" type: DT_INT32 } output_arg { name: "toint32_2" type: DT_INT32 } output_arg { name: "toint32_3" type: DT_INT32 } output_arg { name: "parsesingleexample_parsesingleexample" type: DT_FLOAT } output_arg { name: "toint32_4" type: DT_INT32 } output_arg { name: "toint32_5" type: DT_INT32 } description: "Wrapper for passing nested structures to and from tf.data functions." } node_def { name: "ParseSingleExample/Const" op: "Const" device: "/device:CPU:0" attr { key: "dtype" value { type: DT_INT64 } } attr { key: "value" value { tensor { dtype: DT_INT64 tensor_shape { dim { } } } } } } node_def { name: "ParseSingleExample/Const_1" op: "Const" device: "/device:CPU:0" attr { key: "dtype" value { type: DT_INT64 } } attr { key: "value" value { tensor { dtype: DT_INT64 tensor_shape { dim { } } } } } } node_def { name: "ParseSingleExample/Const_2" op: "Const" device: "/device:CPU:0" attr { key: "dtype" value { type: DT_INT64 } } attr { key: "value" value { tensor { dtype: DT_INT64 tensor_shape { dim { } } } } } } node_def { name: "ParseSingleExample/Const_3" op: "Const" device: "/device:CPU:0" attr { key: "dtype" value { type: DT_INT64 } } attr { key: "value" value { tensor { dtype: DT_INT64 tensor_shape { dim { } } } } } } node_def { name: "ParseSingleExample/Const_4" op: "Const" device: "/device:CPU:0" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node_def { name: "ParseSingleExample/Const_5" op: "Const" device: "/device:CPU:0" attr { key: "dtype" value { type: DT_INT64 } } attr { key: "value" value { tensor { dtype: DT_INT64 tensor_shape { dim { } } } } } } node_def { name: "ParseSingleExample/Const_6" op: "Const" device: "/device:CPU:0" attr { key: "dtype" value { type: DT_INT64 } } attr { key: "value" value { tensor { dtype: DT_INT64 tensor_shape { dim { } } } } } } node_def { name: "ParseSingleExample/ParseSingleExample" op: "ParseSingleExample" input: "arg0" input: "ParseSingleExample/Const:output:0" input: "ParseSingleExample/Const_1:output:0" input: "ParseSingleExample/Const_2:output:0" input: "ParseSingleExample/Const_3:output:0" input: "ParseSingleExample/Const_4:output:0" input: "ParseSingleExample/Const_5:output:0" input: "ParseSingleExample/Const_6:output:0" device: "/device:CPU:0" attr { key: "Tdense" value { list { type: DT_INT64 type: DT_INT64 type: DT_INT64 type: DT_INT64 type: DT_FLOAT type: DT_INT64 type: DT_INT64 } } } attr { key: "dense_keys" value { list { s: "input_ids" s: "input_mask" s: "masked_lm_ids" s: "masked_lm_positions" s: "masked_lm_weights" s: "next_sentence_labels" s: "segment_ids" } } } attr { key: "dense_shapes" value { list { shape { dim { size: 128 } } shape { dim { size: 128 } } shape { dim { size: 20 } } shape { dim { size: 20 } } shape { dim { size: 20 } } shape { dim { size: 1 } } shape { dim { size: 128 } } } } } attr { key: "num_sparse" value { i: 0 } } attr { key: "sparse_keys" value { list { } } } attr { key: "sparse_types" value { list { } } } } node_def { name: "ToInt32" op: "Cast" input: "ParseSingleExample/ParseSingleExample:dense_values:0" device: "/device:CPU:0" attr { key: "DstT" value { type: DT_INT32 } } attr { key: "SrcT" value { type: DT_INT64 } } attr { key: "Truncate" value { b: false } } } node_def { name: "ToInt32_1" op: "Cast" input: "ParseSingleExample/ParseSingleExample:dense_values:1" device: "/device:CPU:0" attr { key: "DstT" value { type: DT_INT32 } } attr { key: "SrcT" value { type: DT_INT64 } } attr { key: "Truncate" value { b: false } } } node_def { name: "ToInt32_2" op: "Cast" input: "ParseSingleExample/ParseSingleExample:dense_values:2" device: "/device:CPU:0" attr { key: "DstT" value { type: DT_INT32 } } attr { key: "SrcT" value { type: DT_INT64 } } attr { key: "Truncate" value { b: false } } } node_def { name: "ToInt32_3" op: "Cast" input: "ParseSingleExample/ParseSingleExample:dense_values:3" device: "/device:CPU:0" attr { key: "DstT" value { type: DT_INT32 } } attr { key: "SrcT" value { type: DT_INT64 } } attr { key: "Truncate" value { b: false } } } node_def { name: "ToInt32_4" op: "Cast" input: "ParseSingleExample/ParseSingleExample:dense_values:5" device: "/device:CPU:0" attr { key: "DstT" value { type: DT_INT32 } } attr { key: "SrcT" value { type: DT_INT64 } } attr { key: "Truncate" value { b: false } } } node_def { name: "ToInt32_5" op: "Cast" input: "ParseSingleExample/ParseSingleExample:dense_values:6" device: "/device:CPU:0" attr { key: "DstT" value { type: DT_INT32 } } attr { key: "SrcT" value { type: DT_INT64 } } attr { key: "Truncate" value { b: false } } } ret { key: "parsesingleexample_parsesingleexample" value: "ParseSingleExample/ParseSingleExample:dense_values:4" } ret { key: "toint32" value: "ToInt32:y:0" } ret { key: "toint32_1" value: "ToInt32_1:y:0" } ret { key: "toint32_2" value: "ToInt32_2:y:0" } ret { key: "toint32_3" value: "ToInt32_3:y:0" } ret { key: "toint32_4" value: "ToInt32_4:y:0" } ret { key: "toint32_5" value: "ToInt32_5:y:0" } } function { signature { name: "tf_data_structured_function_wrapper_52oRgbbAY9U" input_arg { name: "arg0" type: DT_STRING } output_arg { name: "tfrecorddataset" type: DT_VARIANT } description: "Wrapper for passing nested structures to and from tf.data functions." is_stateful: true } node_def { name: "compression_type" op: "Const" device: "/device:CPU:0" attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "" } } } } node_def { name: "buffer_size" op: "Const" device: "/device:CPU:0" attr { key: "dtype" value { type: DT_INT64 } } attr { key: "value" value { tensor { dtype: DT_INT64 tensor_shape { } int64_val: 262144 } } } } node_def { name: "TFRecordDataset" op: "TFRecordDataset" input: "arg0" input: "compression_type:output:0" input: "buffer_size:output:0" device: "/device:CPU:0" } ret { key: "tfrecorddataset" value: "TFRecordDataset:handle:0" } } function { signature { name: "tf_data_structured_function_wrapper_rn3e6kArW78" input_arg { name: "arg0" type: DT_STRING } output_arg { name: "flatmapdataset" type: DT_VARIANT } description: "Wrapper for passing nested structures to and from tf.data functions." is_stateful: true } node_def { name: "flat_filenames/shape" op: "Const" device: "/device:CPU:0" attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: -1 } } } } node_def { name: "flat_filenames" op: "Reshape" input: "arg0" input: "flat_filenames/shape:output:0" device: "/device:CPU:0" attr { key: "T" value { type: DT_STRING } } attr { key: "Tshape" value { type: DT_INT32 } } } node_def { name: "TensorSliceDataset" op: "TensorSliceDataset" input: "flat_filenames:output:0" device: "/device:CPU:0" attr { key: "Toutput_types" value { list { type: DT_STRING } } } attr { key: "output_shapes" value { list { shape { } } } } } node_def { name: "FlatMapDataset" op: "FlatMapDataset" input: "TensorSliceDataset:handle:0" device: "/device:CPU:0" attr { key: "Targuments" value { list { } } } attr { key: "f" value { func { name: "tf_data_structured_function_wrapper_52oRgbbAY9U" } } } attr { key: "output_shapes" value { list { shape { } } } } attr { key: "output_types" value { list { type: DT_STRING } } } } ret { key: "flatmapdataset" value: "FlatMapDataset:handle:0" } } } versions { producer: 27 min_consumer: 12 }